lexer.cpp -- CLexerの実体を実装。

sublexer.cpp -- CLexerから作成され、実際の解析を行なう小さなlexerを実装。
lexeme_impl.h -- lexeme.hに定義した抽象インターフェースを基にして各lexemeを定義する。

diff --git a/GPATH b/GPATH
index e750c10..78f5c79 100644 (file)
Binary files a/GPATH and b/GPATH differ

diff --git a/GRTAGS b/GRTAGS
index 9aa027a..9d80861 100644 (file)
Binary files a/GRTAGS and b/GRTAGS differ

diff --git a/GSYMS b/GSYMS
index e8e12c8..9e81720 100644 (file)
Binary files a/GSYMS and b/GSYMS differ

diff --git a/GTAGS b/GTAGS
index 8123556..b624260 100644 (file)
Binary files a/GTAGS and b/GTAGS differ

diff --git a/Makefile b/Makefile
index d9afd1f..2e39536 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -96,7 +96,7 @@ CC = gcc
 CCDEPMODE = depmode=gcc3
 CFLAGS = -g -O2
 CPP = gcc -E
-CPPFLAGS = -Wall
+CPPFLAGS = 
 CXX = g++
 CXXDEPMODE = depmode=gcc3
 CXXFLAGS = -g -O2
@@ -175,9 +175,12 @@ target_alias =
 top_build_prefix = 
 top_builddir = .
 top_srcdir = .
+AM_CPPFLAGS = -Wall
 SUBDIRS = . test
 INCLUDES = -l$(top_srcdir)
-utakata_SOURCES = main.cpp vm.cpp operand.cpp cpu.cpp utf8.cpp TextArrayFormat.cpp utf8_string.cpp
+utakata_SOURCES = main.cpp vm.cpp operand.cpp cpu.cpp utf8.cpp \
+       TextArrayFormat.cpp utf8_string.cpp
+
 all: config.h
        $(MAKE) $(AM_MAKEFLAGS) all-recursive
 

diff --git a/Makefile.am b/Makefile.am
index c712b51..ec5b31b 100755 (executable)
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,8 @@
+AM_CPPFLAGS = -Wall
 SUBDIRS = . test
 INCLUDES = -l$(top_srcdir)
 
 bin_PROGRAMS = utakata
 
-utakata_SOURCES = main.cpp vm.cpp operand.cpp cpu.cpp utf8.cpp TextArrayFormat.cpp utf8_string.cpp
+utakata_SOURCES = main.cpp vm.cpp operand.cpp cpu.cpp utf8.cpp \
+       TextArrayFormat.cpp utf8_string.cpp

diff --git a/Makefile.in b/Makefile.in
index 2ef9055..a811fc2 100644 (file)
--- a/Makefile.in
+++ b/Makefile.in
@@ -175,9 +175,12 @@ target_alias = @target_alias@
 top_build_prefix = @top_build_prefix@
 top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
+AM_CPPFLAGS = -Wall
 SUBDIRS = . test
 INCLUDES = -l$(top_srcdir)
-utakata_SOURCES = main.cpp vm.cpp operand.cpp cpu.cpp utf8.cpp TextArrayFormat.cpp utf8_string.cpp
+utakata_SOURCES = main.cpp vm.cpp operand.cpp cpu.cpp utf8.cpp \
+       TextArrayFormat.cpp utf8_string.cpp
+
 all: config.h
        $(MAKE) $(AM_MAKEFLAGS) all-recursive
 

diff --git a/config.log b/config.log
index c05673d..02c6796 100644 (file)
--- a/config.log
+++ b/config.log
@@ -12,9 +12,9 @@ generated by GNU Autoconf 2.63.  Invocation command line was
 
 hostname = localhost
 uname -m = i686
-uname -r = 2.6.28-gentoo-r5
+uname -r = 2.6.29-gentoo-r5
 uname -s = Linux
-uname -v = #2 SMP Sun May 24 20:35:46 JST 2009
+uname -v = #5 SMP Sat Jun 6 17:06:52 JST 2009
 
 /usr/bin/uname -p = Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz
 /bin/uname -X     = unknown
@@ -28,14 +28,12 @@ uname -v = #2 SMP Sun May 24 20:35:46 JST 2009
 /bin/universe          = unknown
 
 PATH: /usr/local/bin
-PATH: /usr/local/sbin
-PATH: /usr/sbin
-PATH: /sbin
-PATH: /bin
-PATH: /usr/local/bin
 PATH: /usr/bin
-PATH: /usr/local/X11R6/bin
-PATH: /usr/X11R6/bin
+PATH: /bin
+PATH: /opt/bin
+PATH: /usr/i486-pc-linux-gnu/gcc-bin/4.1.2
+PATH: /usr/i686-pc-linux-gnu/gcc-bin/4.3.2
+PATH: /usr/qt/3/bin
 
 
 ## ----------- ##
@@ -49,7 +47,7 @@ configure:2005: result: yes
 configure:2030: checking for a thread-safe mkdir -p
 configure:2069: result: /bin/mkdir -p
 configure:2082: checking for gawk
-configure:2098: found /bin/gawk
+configure:2098: found /usr/bin/gawk
 configure:2109: result: gawk
 configure:2120: checking whether make sets $(MAKE)
 configure:2142: result: yes
@@ -58,8 +56,8 @@ configure:2393: found /usr/bin/g++
 configure:2404: result: g++
 configure:2431: checking for C++ compiler version
 configure:2439: g++ --version >&5
-g++ (GCC) 4.1.2 (Gentoo 4.1.2 p1.0.2)
-Copyright (C) 2006 Free Software Foundation, Inc.
+g++ (Gentoo 4.3.2-r3 p1.6, pie-10.1.5) 4.3.2
+Copyright (C) 2008 Free Software Foundation, Inc.
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
@@ -67,9 +65,9 @@ configure:2443: $? = 0
 configure:2450: g++ -v >&5
 Using built-in specs.
 Target: i686-pc-linux-gnu
-Configured with: /var/tmp/portage/sys-devel/gcc-4.1.2/work/gcc-4.1.2/configure --prefix=/usr --bindir=/usr/i686-pc-linux-gnu/gcc-bin/4.1.2 --includedir=/usr/lib/gcc/i686-pc-linux-gnu/4.1.2/include --datadir=/usr/share/gcc-data/i686-pc-linux-gnu/4.1.2 --mandir=/usr/share/gcc-data/i686-pc-linux-gnu/4.1.2/man --infodir=/usr/share/gcc-data/i686-pc-linux-gnu/4.1.2/info --with-gxx-include-dir=/usr/lib/gcc/i686-pc-linux-gnu/4.1.2/include/g++-v4 --host=i686-pc-linux-gnu --build=i686-pc-linux-gnu --disable-altivec --enable-nls --without-included-gettext --with-system-zlib --disable-checking --disable-werror --enable-secureplt --disable-libunwind-exceptions --disable-multilib --enable-libmudflap --disable-libssp --disable-libgcj --with-arch=i686 --enable-languages=c,c++,treelang,fortran --enable-shared --enable-threads=posix --enable-__cxa_atexit --enable-clocale=gnu
+Configured with: /var/tmp/portage/sys-devel/gcc-4.3.2-r3/work/gcc-4.3.2/configure --prefix=/usr --bindir=/usr/i686-pc-linux-gnu/gcc-bin/4.3.2 --includedir=/usr/lib/gcc/i686-pc-linux-gnu/4.3.2/include --datadir=/usr/share/gcc-data/i686-pc-linux-gnu/4.3.2 --mandir=/usr/share/gcc-data/i686-pc-linux-gnu/4.3.2/man --infodir=/usr/share/gcc-data/i686-pc-linux-gnu/4.3.2/info --with-gxx-include-dir=/usr/lib/gcc/i686-pc-linux-gnu/4.3.2/include/g++-v4 --host=i686-pc-linux-gnu --build=i686-pc-linux-gnu --disable-altivec --disable-fixed-point --enable-nls --without-included-gettext --with-system-zlib --disable-checking --disable-werror --enable-secureplt --disable-multilib --enable-libmudflap --disable-libssp --enable-libgomp --disable-libgcj --with-arch=i686 --enable-languages=c,c++,treelang,fortran --enable-shared --enable-threads=posix --enable-__cxa_atexit --enable-clocale=gnu --with-bugurl=http://bugs.gentoo.org/ --with-pkgversion='Gentoo 4.3.2-r3 p1.6, pie-10.1.5'
 Thread model: posix
-gcc version 4.1.2 (Gentoo 4.1.2 p1.0.2)
+gcc version 4.3.2 (Gentoo 4.3.2-r3 p1.6, pie-10.1.5) 
 configure:2454: $? = 0
 configure:2461: g++ -V >&5
 g++: '-V' option must have argument
@@ -109,8 +107,8 @@ configure:3175: found /usr/bin/gcc
 configure:3186: result: gcc
 configure:3418: checking for C compiler version
 configure:3426: gcc --version >&5
-gcc (GCC) 4.1.2 (Gentoo 4.1.2 p1.0.2)
-Copyright (C) 2006 Free Software Foundation, Inc.
+gcc (Gentoo 4.3.2-r3 p1.6, pie-10.1.5) 4.3.2
+Copyright (C) 2008 Free Software Foundation, Inc.
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
@@ -118,9 +116,9 @@ configure:3430: $? = 0
 configure:3437: gcc -v >&5
 Using built-in specs.
 Target: i686-pc-linux-gnu
-Configured with: /var/tmp/portage/sys-devel/gcc-4.1.2/work/gcc-4.1.2/configure --prefix=/usr --bindir=/usr/i686-pc-linux-gnu/gcc-bin/4.1.2 --includedir=/usr/lib/gcc/i686-pc-linux-gnu/4.1.2/include --datadir=/usr/share/gcc-data/i686-pc-linux-gnu/4.1.2 --mandir=/usr/share/gcc-data/i686-pc-linux-gnu/4.1.2/man --infodir=/usr/share/gcc-data/i686-pc-linux-gnu/4.1.2/info --with-gxx-include-dir=/usr/lib/gcc/i686-pc-linux-gnu/4.1.2/include/g++-v4 --host=i686-pc-linux-gnu --build=i686-pc-linux-gnu --disable-altivec --enable-nls --without-included-gettext --with-system-zlib --disable-checking --disable-werror --enable-secureplt --disable-libunwind-exceptions --disable-multilib --enable-libmudflap --disable-libssp --disable-libgcj --with-arch=i686 --enable-languages=c,c++,treelang,fortran --enable-shared --enable-threads=posix --enable-__cxa_atexit --enable-clocale=gnu
+Configured with: /var/tmp/portage/sys-devel/gcc-4.3.2-r3/work/gcc-4.3.2/configure --prefix=/usr --bindir=/usr/i686-pc-linux-gnu/gcc-bin/4.3.2 --includedir=/usr/lib/gcc/i686-pc-linux-gnu/4.3.2/include --datadir=/usr/share/gcc-data/i686-pc-linux-gnu/4.3.2 --mandir=/usr/share/gcc-data/i686-pc-linux-gnu/4.3.2/man --infodir=/usr/share/gcc-data/i686-pc-linux-gnu/4.3.2/info --with-gxx-include-dir=/usr/lib/gcc/i686-pc-linux-gnu/4.3.2/include/g++-v4 --host=i686-pc-linux-gnu --build=i686-pc-linux-gnu --disable-altivec --disable-fixed-point --enable-nls --without-included-gettext --with-system-zlib --disable-checking --disable-werror --enable-secureplt --disable-multilib --enable-libmudflap --disable-libssp --enable-libgomp --disable-libgcj --with-arch=i686 --enable-languages=c,c++,treelang,fortran --enable-shared --enable-threads=posix --enable-__cxa_atexit --enable-clocale=gnu --with-bugurl=http://bugs.gentoo.org/ --with-pkgversion='Gentoo 4.3.2-r3 p1.6, pie-10.1.5'
 Thread model: posix
-gcc version 4.1.2 (Gentoo 4.1.2 p1.0.2)
+gcc version 4.3.2 (Gentoo 4.3.2-r3 p1.6, pie-10.1.5) 
 configure:3441: $? = 0
 configure:3448: gcc -V >&5
 gcc: '-V' option must have argument
@@ -139,12 +137,12 @@ configure:3753: $? = 0
 configure:3776: result: none needed
 configure:3796: checking dependency style of gcc
 configure:3887: result: gcc3
-configure:3914: checking how to run the C preprocessor
-configure:3954: gcc -E  conftest.c
-configure:3961: $? = 0
-configure:3992: gcc -E  conftest.c
+configure:3915: checking how to run the C preprocessor
+configure:3955: gcc -E  conftest.c
+configure:3962: $? = 0
+configure:3993: gcc -E  conftest.c
 conftest.c:8:28: error: ac_nonexistent.h: No such file or directory
-configure:3999: $? = 1
+configure:4000: $? = 1
 configure: failed program was:
 | /* confdefs.h.  */
 | #define PACKAGE_NAME "utakata"
@@ -154,12 +152,12 @@ configure: failed program was:
 | #define PACKAGE_BUGREPORT "BUG-REPORT-ADDRESS"
 | /* end confdefs.h.  */
 | #include <ac_nonexistent.h>
-configure:4032: result: gcc -E
-configure:4061: gcc -E  conftest.c
-configure:4068: $? = 0
-configure:4099: gcc -E  conftest.c
+configure:4033: result: gcc -E
+configure:4062: gcc -E  conftest.c
+configure:4069: $? = 0
+configure:4100: gcc -E  conftest.c
 conftest.c:8:28: error: ac_nonexistent.h: No such file or directory
-configure:4106: $? = 1
+configure:4107: $? = 1
 configure: failed program was:
 | /* confdefs.h.  */
 | #define PACKAGE_NAME "utakata"
@@ -169,65 +167,65 @@ configure: failed program was:
 | #define PACKAGE_BUGREPORT "BUG-REPORT-ADDRESS"
 | /* end confdefs.h.  */
 | #include <ac_nonexistent.h>
-configure:4146: checking for grep that handles long lines and -e
-configure:4206: result: /bin/grep
-configure:4211: checking for egrep
-configure:4275: result: /bin/grep -E
-configure:4280: checking for ANSI C header files
-configure:4310: gcc -c -g -O2  conftest.c >&5
-configure:4317: $? = 0
-configure:4416: gcc -o conftest -g -O2   conftest.c  >&5
-configure:4420: $? = 0
-configure:4426: ./conftest
-configure:4430: $? = 0
-configure:4448: result: yes
-configure:4472: checking for sys/types.h
-configure:4493: gcc -c -g -O2  conftest.c >&5
-configure:4500: $? = 0
-configure:4517: result: yes
-configure:4472: checking for sys/stat.h
-configure:4493: gcc -c -g -O2  conftest.c >&5
-configure:4500: $? = 0
-configure:4517: result: yes
-configure:4472: checking for stdlib.h
-configure:4493: gcc -c -g -O2  conftest.c >&5
-configure:4500: $? = 0
-configure:4517: result: yes
-configure:4472: checking for string.h
-configure:4493: gcc -c -g -O2  conftest.c >&5
-configure:4500: $? = 0
-configure:4517: result: yes
-configure:4472: checking for memory.h
-configure:4493: gcc -c -g -O2  conftest.c >&5
-configure:4500: $? = 0
-configure:4517: result: yes
-configure:4472: checking for strings.h
-configure:4493: gcc -c -g -O2  conftest.c >&5
-configure:4500: $? = 0
-configure:4517: result: yes
-configure:4472: checking for inttypes.h
-configure:4493: gcc -c -g -O2  conftest.c >&5
-configure:4500: $? = 0
-configure:4517: result: yes
-configure:4472: checking for stdint.h
-configure:4493: gcc -c -g -O2  conftest.c >&5
-configure:4500: $? = 0
-configure:4517: result: yes
-configure:4472: checking for unistd.h
-configure:4493: gcc -c -g -O2  conftest.c >&5
-configure:4500: $? = 0
-configure:4517: result: yes
-configure:4531: checking for stdbool.h that conforms to C99
-configure:4626: gcc -c -g -O2  conftest.c >&5
-configure:4633: $? = 0
-configure:4648: result: yes
-configure:4650: checking for _Bool
-configure:4678: gcc -c -g -O2  conftest.c >&5
-configure:4685: $? = 0
-configure:4712: gcc -c -g -O2  conftest.c >&5
+configure:4147: checking for grep that handles long lines and -e
+configure:4207: result: /bin/grep
+configure:4212: checking for egrep
+configure:4276: result: /bin/grep -E
+configure:4281: checking for ANSI C header files
+configure:4311: gcc -c -g -O2  conftest.c >&5
+configure:4318: $? = 0
+configure:4417: gcc -o conftest -g -O2   conftest.c  >&5
+configure:4421: $? = 0
+configure:4427: ./conftest
+configure:4431: $? = 0
+configure:4449: result: yes
+configure:4473: checking for sys/types.h
+configure:4494: gcc -c -g -O2  conftest.c >&5
+configure:4501: $? = 0
+configure:4518: result: yes
+configure:4473: checking for sys/stat.h
+configure:4494: gcc -c -g -O2  conftest.c >&5
+configure:4501: $? = 0
+configure:4518: result: yes
+configure:4473: checking for stdlib.h
+configure:4494: gcc -c -g -O2  conftest.c >&5
+configure:4501: $? = 0
+configure:4518: result: yes
+configure:4473: checking for string.h
+configure:4494: gcc -c -g -O2  conftest.c >&5
+configure:4501: $? = 0
+configure:4518: result: yes
+configure:4473: checking for memory.h
+configure:4494: gcc -c -g -O2  conftest.c >&5
+configure:4501: $? = 0
+configure:4518: result: yes
+configure:4473: checking for strings.h
+configure:4494: gcc -c -g -O2  conftest.c >&5
+configure:4501: $? = 0
+configure:4518: result: yes
+configure:4473: checking for inttypes.h
+configure:4494: gcc -c -g -O2  conftest.c >&5
+configure:4501: $? = 0
+configure:4518: result: yes
+configure:4473: checking for stdint.h
+configure:4494: gcc -c -g -O2  conftest.c >&5
+configure:4501: $? = 0
+configure:4518: result: yes
+configure:4473: checking for unistd.h
+configure:4494: gcc -c -g -O2  conftest.c >&5
+configure:4501: $? = 0
+configure:4518: result: yes
+configure:4532: checking for stdbool.h that conforms to C99
+configure:4627: gcc -c -g -O2  conftest.c >&5
+configure:4634: $? = 0
+configure:4649: result: yes
+configure:4651: checking for _Bool
+configure:4679: gcc -c -g -O2  conftest.c >&5
+configure:4686: $? = 0
+configure:4713: gcc -c -g -O2  conftest.c >&5
 conftest.c: In function 'main':
 conftest.c:54: error: expected expression before ')' token
-configure:4719: $? = 1
+configure:4720: $? = 1
 configure: failed program was:
 | /* confdefs.h.  */
 | #define PACKAGE_NAME "utakata"
@@ -287,18 +285,18 @@ configure: failed program was:
 |   ;
 |   return 0;
 | }
-configure:4742: result: yes
-configure:4761: checking for inline
-configure:4787: gcc -c -g -O2  conftest.c >&5
-configure:4794: $? = 0
-configure:4812: result: inline
-configure:4831: checking for size_t
-configure:4859: gcc -c -g -O2  conftest.c >&5
-configure:4866: $? = 0
-configure:4893: gcc -c -g -O2  conftest.c >&5
+configure:4743: result: yes
+configure:4762: checking for inline
+configure:4788: gcc -c -g -O2  conftest.c >&5
+configure:4795: $? = 0
+configure:4813: result: inline
+configure:4832: checking for size_t
+configure:4860: gcc -c -g -O2  conftest.c >&5
+configure:4867: $? = 0
+configure:4894: gcc -c -g -O2  conftest.c >&5
 conftest.c: In function 'main':
 conftest.c:56: error: expected expression before ')' token
-configure:4900: $? = 1
+configure:4901: $? = 1
 configure: failed program was:
 | /* confdefs.h.  */
 | #define PACKAGE_NAME "utakata"
@@ -360,8 +358,8 @@ configure: failed program was:
 |   ;
 |   return 0;
 | }
-configure:4923: result: yes
-configure:5063: creating ./config.status
+configure:4924: result: yes
+configure:5064: creating ./config.status
 
 ## ---------------------- ##
 ## Running config.status. ##
@@ -559,75 +557,3 @@ target_alias=''
 #define HAVE_STDBOOL_H 1
 
 configure: exit 0
-
-## ---------------------- ##
-## Running config.status. ##
-## ---------------------- ##
-
-This file was extended by utakata config.status 0.0.1, which was
-generated by GNU Autoconf 2.63.  Invocation command line was
-
-  CONFIG_FILES    = 
-  CONFIG_HEADERS  = 
-  CONFIG_LINKS    = 
-  CONFIG_COMMANDS = 
-  $ ./config.status test/Makefile depfiles
-
-on localhost
-
-config.status:776: creating test/Makefile
-config.status:1050: executing depfiles commands
-
-## ---------------------- ##
-## Running config.status. ##
-## ---------------------- ##
-
-This file was extended by utakata config.status 0.0.1, which was
-generated by GNU Autoconf 2.63.  Invocation command line was
-
-  CONFIG_FILES    = 
-  CONFIG_HEADERS  = 
-  CONFIG_LINKS    = 
-  CONFIG_COMMANDS = 
-  $ ./config.status test/Makefile depfiles
-
-on localhost
-
-config.status:776: creating test/Makefile
-config.status:1050: executing depfiles commands
-
-## ---------------------- ##
-## Running config.status. ##
-## ---------------------- ##
-
-This file was extended by utakata config.status 0.0.1, which was
-generated by GNU Autoconf 2.63.  Invocation command line was
-
-  CONFIG_FILES    = 
-  CONFIG_HEADERS  = 
-  CONFIG_LINKS    = 
-  CONFIG_COMMANDS = 
-  $ ./config.status test/Makefile depfiles
-
-on localhost
-
-config.status:776: creating test/Makefile
-config.status:1050: executing depfiles commands
-
-## ---------------------- ##
-## Running config.status. ##
-## ---------------------- ##
-
-This file was extended by utakata config.status 0.0.1, which was
-generated by GNU Autoconf 2.63.  Invocation command line was
-
-  CONFIG_FILES    = 
-  CONFIG_HEADERS  = 
-  CONFIG_LINKS    = 
-  CONFIG_COMMANDS = 
-  $ ./config.status Makefile depfiles
-
-on localhost
-
-config.status:776: creating Makefile
-config.status:1050: executing depfiles commands

diff --git a/configure b/configure
index eda120b..a5ce00b 100755 (executable)
--- a/configure
+++ b/configure
@@ -3900,6 +3900,7 @@ fi
 
 
 
+
 # Checks for libraries.
 
 # Checks for header files.

diff --git a/configure.in b/configure.in
index 8793754..c2d32dd 100644 (file)
--- a/configure.in
+++ b/configure.in
@@ -11,6 +11,7 @@ AM_INIT_AUTOMAKE(utakata, 0.0.1, derutakayu@gmail.com)
 AC_PROG_CXX
 AC_PROG_CC
 
+
 # Checks for libraries.
 
 # Checks for header files.

diff --git a/lexeme.h b/lexeme.h
index c8dbab9..2c2e23c 100644 (file)
--- a/lexeme.h
+++ b/lexeme.h
@@ -2,446 +2,41 @@
 #define _LEXEME_H_
 
 #include "smart_ptr.h"
+#include "utf8_string.h"
 
 namespace utakata {
 
     namespace lexeme {
 
-        class utakata::utf8_string::CUTF8InputStream;
-        class utakata::utf8_string::CUTF8String;
-        class CLexeme;
-
-        // lexer名前空間で使用されるそれぞれの終端記号と
-        // それらを解析するための関数オブジェクトを定義する。
-
-        struct CIdentityLexer
-        {
-            /*
-              <identifier>を解釈するためのfunctor。
-              <identifier>は次のBNF構文で定義される。
-              
-              <identifier> - <initial> <subsequent>*
-                           | <peculiar identifier>
-             */
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CInitialLexer
-        {
-            /*
-              <initial>を解釈するためのfunctor。
-              <initial>は次のBNF構文で定義される。
-              
-              <initial> - <constituent> | <special initial>
-                        | <inline hex escape>
-            */
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CConstituentLexer
-        {
-            /*
-              <constituent>を解釈するためのfunctor。
-              <constituent>は次のBNF構文で定義される。
-
-              <constituent> - <letter>
-              | 〈any character whose Unicode scalar value is greater than
-                  127, and whose category is Lu, Ll, Lt, Lm, Lo, Mn,
-                  Nl, No, Pd, Pc, Po, Sc, Sm, Sk, So, or Co〉
-            */
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CLetterlexer
-        {
-            /*
-              <letter>を解釈するためのfunctor。
-              <letter>は次のBNF構文で定義される。
-
-              <letter> - a | b | c | ... | z
-                       | A | B | C | ... | Z
-            */
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CSpecialInitialLexer
-        {
-            /*
-              <special initial>を解釈するためのfunctor。
-              <special initial>は次のBNF構文で定義される。
-              <special initial> - ! | $ | % | & | * | / | : | < | =
-                                | > | ? | ^ | _ | ~
-
-            */
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CDigitlexer
-        {
-            /*
-              <digit>を解釈するためのfunctor。
-              <digit>は次のBNF構文で定義される。
-              <digit> - 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9
-            */
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CHexDigitLexer
-        {
-            /*
-              <hex digit>を解釈するためのfunctor。
-              <hex digit>は次のBNF構文で定義される。
-              <hex digit> - <digit>
-                          | a | A | b | B | c | C | d | D | e | E | f
-                          | F
-            */
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CSpecialSubsequentLexer
-        {
-            /*
-              <special subsequent>を解釈するためのfunctor。
-              <special subsequent>は次のBNF構文で定義される。
-              <special subsequent> → + | - | . | @
-            */
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CInlineHexEscapeLexer
-        {
-            /*
-              <inline hex escape>を解釈するためのfunctor
-              <inline hex escape>は次のBNF構文で定義される。
-              <inline hex escape> - \x<hex scalar value>;
-            */
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CHexScalarValueLexer
-        {
-            //<hex scalar value>を解釈するためのfunctor。
-            //<hex scalar value>は次のBNF構文で定義される。
-            //<hex scalar value> - <hex digit>+
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CBooleanLexer
-        {
-            // <boolean>を解釈するためのfunctor。
-            // <boolean>は次のBNF構文で定義される。
-            // <boolean> - #t | #T | #f | #F
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CCharactorLexer
-        {
-            // <charactor>を解釈するためのfunctor。
-            // <charactor>は次のBNF構文で定義される。
-            // <character> - #\<any character>
-            //                 | #\<character name>
-            //                 | #\x<hex scalar value>
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CCharactorNameLexer
-        {
-//             <character name>を解釈するためのfunctor。
-//             <charactor name>は次のBNF構文で定義される。
-//             <character name> - nul | alarm | backspace | tab
-//             | linefeed | newline | vtab | page | return
-//             | esc | space | delete
-
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CStringlexer
-        {
-//             <string>を解釈するためのfunctor。
-//             <string>は次のBNF構文で定義される。
-//             <string> - " <string element>* "
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CStringElementLexer
-        {
-//             <string element>を解釈するためのfunctor。
-//             <string element>は次のBNF構文で定義される。
-//             <string element> - <any character other than " or \>
-//          | \a | \b | \t | \n | \v | \f | \r
-//          | \" | \\
-//          | \<intraline whitespace>*<line ending>
-//             <intraline whitespace>*
-//          | <inline hex escape>
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CIntralineWhitespaceLexer
+        class ILexeme
         {
-//             <intraline whitespace>を解釈するためのfunctor。
-//             <intraline whitespace>は次のBNF構文で定義される。
-//             <intraline whitespace> - <character tabulation>
-//             | <any character whose category is Zs>
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CDelimiterlexer
-        {
-//             <delimiter>を解釈するためのfunctor。
-//             <delimiter>は次のBNF構文で定義される。
-//             <delimiter> - ( | ) | [ | ] | " | ; | #
-//          | <whitespace>
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CWhitespaceLexer
-        {
-//             <whitespace>を解釈するためのfunctor。
-//             <whitespace>は次のBNF構文で定義される。
-//             <whitespace> - <character tabulation>
-//             | <linefeed> | <line tabulation> | <form feed>
-//             | <carriage return> | <next line>
-//             | <any character whose category is Zs, Zl, or Zp>
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CLineEndingLexer
-        {
-//             <line ending>を解釈するためのfunctor。
-//             <line ending>は次のBNF構文で定義される。
-//             <line ending> - <linefeed> | <carriage return>
-//             | <carriage return> <linefeed> | <next line>
-//             | <carriage return> <next line> | <line separator>
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CCommentLexer
-        {
-//             <comment>を解釈するためのfunctor。
-//             <comment>は次のBNF構文で定義される。
-//             <comment> - ; 〈all subsequent characters up to a
-//             <line ending> or <paragraph separator>〉
-//             | <nested comment>
-//             | #; <interlexeme space> <datum>
-//             | #!r6rs
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CNestedCommentLexer
-        {
-//             <nested comment>を解釈するためのfunctor。
-//             <nested comment>は次のBNF構文で定義される。
-//             <nested comment> - #| <comment text>
-//             <comment cont>* |#
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CCommentTextLexer
-        {
-//             <comment text>を解釈するためのfunctor。
-//             <comment text>は次のBNF構文で定義される。
-//             <comment text> - 〈character sequence not containing
-//             #| or |#〉
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CCommentContLexer
-        {
-//             <comment cont>を解釈するためのfunctor。
-//             <comment cont>は次のBNF構文で定義される。
-//             <comment cont> - <nested comment> <comment text>
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-
-        struct CNumberLexer
-        {
-//             <number>を解釈するためのfunctor。
-//             <number>は次のBNF構文で定義される。
-//             <number> - <num 2> | <num 8>
-//             | <num 10> | <num 16>
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CNum2Lexer
-        {
-//             <num 2>を解釈するためのfunctor。
-//             <num 2>は次のBNF構文で定義される。
-//             <num R> - <prefix R> <complex R>
-//             <complex 2> - <real 2> | <real 2> @ <real 2>
-//             | <real 2> + <ureal 2> i | <real 2> - <ureal 2> i
-//             | <real 2> + <naninf> i | <real 2> - <naninf> i
-//             | <real 2> + i | <real 2> - i
-//             | + <ureal 2> i | - <ureal 2> i
-//             | + <naninf> i | - <naninf> i
-//             | + i | - i
-//             <real 2> - <sign> <ureal 2>
-//             | + <naninf> | - <naninf>
-//             <naninf> - nan.0 | inf.0
-//             <ureal 2> - <uinteger 2>
-//             | <uinteger 2> / <uinteger 2>
-//             | <decimal 2> <mantissa width>
-//             <decimal 10> - <uinteger 10> <suffix>
-//             | . <digit 10>+ <suffix>
-//             | <digit 10>+ . <digit 10>* <suffix>
-//             <uinteger 2> - <digit 2>+
-//             <prefix 2> - <radix 2> <exactness>
-//             | <exactness> <radix 2>
-
-//             <suffix> - <empty>
-//             | <exponent marker> <sign> <digit 10>+
-//             <exponent marker> - e | E | s | S | f | F
-//             | d | D | l | L
-//             <mantissa width> - <empty>
-//             | | <digit 10>+
-//             <sign> - <empty> | + | -
-//             <exactness> - <empty>
-//             | #i| #I | #e| #E
-//             <radix 2> - #b| #B
-//             <digit 2> - 0 | 1
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CNum8Lexer
-        {
-//             <num 8>を解釈するためのfunctor。
-//             <num 8>は次のBNF構文で定義される。
-//             <num R> - <prefix R> <complex R>
-//             <complex 8> - <real 8> | <real 8> @ <real 8>
-//             | <real 8> + <ureal 8> i | <real 8> - <ureal 8> i
-//             | <real 8> + <naninf> i | <real 8> - <naninf> i
-//             | <real 8> + i | <real 8> - i
-//             | + <ureal 8> i | - <ureal 8> i
-//             | + <naninf> i | - <naninf> i
-//             | + i | - i
-//             <real 8> - <sign> <ureal 8>
-//             | + <naninf> | - <naninf>
-//             <naninf> - nan.0 | inf.0
-//             <ureal 8> - <uinteger 8>
-//             | <uinteger 8> / <uinteger 8>
-//             | <decimal 8> <mantissa width>
-//             <decimal 10> - <uinteger 10> <suffix>
-//             | . <digit 10>+ <suffix>
-//             | <digit 10>+ . <digit 10>* <suffix>
-//             <uinteger 8> - <digit 8>+
-//             <prefix 8> - <radix 8> <exactness>
-//             | <exactness> <radix 8>
-
-//             <suffix> - <empty>
-//             | <exponent marker> <sign> <digit 10>+
-//             <exponent marker> - e | E | s | S | f | F
-//             | d | D | l | L
-//             <mantissa width> - <empty>
-//             | | <digit 10>+
-//             <sign> - <empty> | + | -
-//             <exactness> - <empty>
-//             | #i| #I | #e| #E
-//             <radix 8> - #b| #B
-//             <digit 8> - 0 | 1
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CNum10Lexer
-        {
-//             <num 10>を解釈するためのfunctor。
-//             <num 10>は次のBNF構文で定義される。
-//             <num R> - <prefix R> <complex R>
-//             <complex 10> - <real 10> | <real 10> @ <real 10>
-//             | <real 10> + <ureal 10> i | <real 10> - <ureal 10> i
-//             | <real 10> + <naninf> i | <real 10> - <naninf> i
-//             | <real 10> + i | <real 10> - i
-//             | + <ureal 10> i | - <ureal 10> i
-//             | + <naninf> i | - <naninf> i
-//             | + i | - i
-//             <real 10> - <sign> <ureal 10>
-//             | + <naninf> | - <naninf>
-//             <naninf> - nan.0 | inf.0
-//             <ureal 10> - <uinteger 10>
-//             | <uinteger 10> / <uinteger 10>
-//             | <decimal 10> <mantissa width>
-//             <decimal 10> - <uinteger 10> <suffix>
-//             | . <digit 10>+ <suffix>
-//             | <digit 10>+ . <digit 10>* <suffix>
-//             <uinteger 10> - <digit 10>+
-//             <prefix 10> - <radix 10> <exactness>
-//             | <exactness> <radix 10>
-
-//             <suffix> - <empty>
-//             | <exponent marker> <sign> <digit 10>+
-//             <exponent marker> - e | E | s | S | f | F
-//             | d | D | l | L
-//             <mantissa width> - <empty>
-//             | | <digit 10>+
-//             <sign> - <empty> | + | -
-//             <exactness> - <empty>
-//             | #i| #I | #e| #E
-//             <radix 10> - #b| #B
-//             <digit 10> - 0 | 1
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);
-        };
-
-        struct CNum16Lexer
-        {
-//             <num 16>を解釈するためのfunctor。
-//             <num 16>は次のBNF構文で定義される。
-//             <num R> - <prefix R> <complex R>
-//             <complex 16> - <real 16> | <real 16> @ <real 16>
-//             | <real 16> + <ureal 16> i | <real 16> - <ureal 16> i
-//             | <real 16> + <naninf> i | <real 16> - <naninf> i
-//             | <real 16> + i | <real 16> - i
-//             | + <ureal 16> i | - <ureal 16> i
-//             | + <naninf> i | - <naninf> i
-//             | + i | - i
-//             <real 16> - <sign> <ureal 16>
-//             | + <naninf> | - <naninf>
-//             <naninf> - nan.0 | inf.0
-//             <ureal 16> - <uinteger 16>
-//             | <uinteger 16> / <uinteger 16>
-//             | <decimal 16> <mantissa width>
-//             <decimal 10> - <uinteger 10> <suffix>
-//             | . <digit 10>+ <suffix>
-//             | <digit 10>+ . <digit 10>* <suffix>
-//             <uinteger 16> - <digit 16>+
-//             <prefix 16> - <radix 16> <exactness>
-//             | <exactness> <radix 16>
-
-//             <suffix> - <empty>
-//             | <exponent marker> <sign> <digit 10>+
-//             <exponent marker> - e | E | s | S | f | F
-//             | d | D | l | L
-//             <mantissa width> - <empty>
-//             | | <digit 10>+
-//             <sign> - <empty> | + | -
-//             <exactness> - <empty>
-//             | #i| #I | #e| #E
-//             <radix 16> - #b| #B
-//             <digit 16> - 0 | 1
-            smart_ptr<CLexeme> operator()(smart_ptr<utakata::utf8_string::CUTF8InputStream>& stream);            
-        };
-
-        //======================================================================
-
-        class CLexeme
-        {
-            // 非終端記号、及び終端記号を表す。
-            // とりあえず最もシンプルな形を取るよ。
+            // 非終端記号、及び終端記号を表すクラス。
+            // それぞれ分割しても問題なかったのだが、それぞれのデータが
+            // 互いに非可換であるが、それぞれ継承してもどうしようもないため、
+            // さしあたって必要となる全ての型を取得できるようにしておき、
+            // それをそのまま取得してもらう、という形にする。
+            // つまりは全体をpimplにしておき、データはそれらから取得するようにするということ。
+            // それぞれを取得するためのインターフェースはこれから作成される。
         public:
-            ILexeme(const utakata::utf8_string::CUTF8String& str, int id);
+            ILexeme();
             virtual ~ILexeme(){}
 
             // 終端記号、非終端記号のIDを取得する。
             int getID() const;
 
-            // そのものを表す文字列を返す。
-            const utakata::utf8_string::CUTF8String& getString() const;
+            // stringのデータ型において、文字列を取得する。
+            smart_ptr<utakata::utf8_string::CUTF8String> getString() const;
 
         private:
-
-            // pimplイディオムを使用する。
-            struct Impl;
-            smart_ptr<Impl> pimpl_;
+            // 必要な型全てをまとめるための構造体。ただし、
+            // 内部のどれか一つだけが有効となっており、これを必要と
+            // しない終端記号のデータも存在する。
+            // 詳しくは各データを解釈する場所にて。
+            struct PImpl;
+            smart_ptr<PImpl> pimpl_;
         };
 
+
     };
 
 };

diff --git a/lexeme_impl.h b/lexeme_impl.h
new file mode 100644 (file)
index 0000000..11ad245
--- /dev/null
+++ b/lexeme_impl.h
@@ -0,0 +1,14 @@
+#ifndef _LEXEME_IMPL_H_
+#define _LEXEME_IMPL_H_
+
+// lexemeの様々な実装を定義する。
+namespace utakata {
+
+    namespace lexeme {
+
+        class ILexeme;
+    };
+
+};
+
+#endif /* _LEXEME_IMPL_H_ */

diff --git a/lexer.cpp b/lexer.cpp
index d0efbb1..47e2095 100644 (file)
--- a/lexer.cpp
+++ b/lexer.cpp
@@ -1,10 +1,13 @@
 #include <iostream>
+#include <assert.h>
 
 #include "lexer.h"
+#include "sublexer.h"
+#include "lexeme.h"
 
 using namespace utakata;
 
-smart_ptr<lexer::CLexeme> lexer::CLexer::lex(smart_ptr<utf8::CUTF8InputStream>& stream)
+smart_ptr<lexeme::ILexeme> lexer::CLexer::lex(smart_ptr<utf8::CUTF8InputStream>& stream)
 {
     // 渡されたCUTF8InputStreamから、1文字ずつ読んでいき、各構文を解釈
     // する。
@@ -14,73 +17,26 @@ smart_ptr<lexer::CLexeme> lexer::CLexer::lex(smart_ptr<utf8::CUTF8InputStream>&
     // まずは何はなくとも1文字読みだす。
 
     // EOF以外の値の場合には、通常の字句解析を行っていく。
-    utf8_string::CUTF8String str();
-    bool first = true;
-    while (!stream->isEOF()) {
+    utf8_string::CUTF8String str;
 
-        // 意味のある文字のみで構成していく。
-
-        utf8_string::CUTF8Char ch(stream->read());
-
-        // 最初の一文字で、大体決定されるため、firstをチェックして
-        // いく。
-        if (first)
-        {
-            first = false;
-
-            // ()[]`',.は構文解析の重要な要素となるため、このまま返す。
-            if (ch.toUTF16Code() == '(' || ch.toUTF16Code() == '[')
-            {
-                return makeStartParen();
-            }
-
-            if (ch.toUTF16Code() == ')' || ch.toUTF16Code() == ']')
-            {
-                return makeEndParen();
-            }
-
-            if (ch.toUTF16Code() == '`')
-            {
-                return makeBackQuote();
-            }
-
-            if (ch.toUTF16Code() == '\'')
-            {
-                return makeQuote();
-            }
+    smart_ptr<sublexer::ISubLexer> sublex(new sublexer::CFirstLex());
+    
+    while (!stream->isEOF() && !sublex.isNull()) {
 
-            if (ch.toUTF16Code() == '.')
-            {
-                return makeDot();
-            }
-
-            if (ch.toUTF16Code() == ',')
-            {
-                return makeComma();
-            }
-
-            if (ch.toUTF16Code() == '"')
-            {
-                // 先頭が"の場合、stringと判断される。
-                return makeString(stream);
-            }
+        // 意味のある文字のみで構成していく。
 
-        }
-        else
+        utf8_string::CUTF8Char ch = stream->peek();
+        smart_ptr<sublexer::ISubLexer> next = sublex->lex(ch, str, stream);
+        if (next.isNull())
         {
-            // それまでに読みだした文字列が固定の文字列かどうか。
-            
-            // 読出した文字がデリミタであるかどうか。
-            if (isDelimiter(ch))
-            {
-                // デリミタ文字である場合、このデリミタ文字に来るまでの間に
-                // 保存した文字列から、実際のデータを作成する。
-                // ここに来るまでにすでに型は決定されているはずなので、
-                // その型を基にして作成する。
-                
-            }
-
-            
+            return sublex->getLexeme();
         }
+        // 次に移れるようにしておく。
+        sublex = next;
+        str += utf8_string::CUTF8Char(stream->read());
     }
+
+    // 何もなく終了してしまった場合、これはエラーとなる。
+    return smart_ptr<lexeme::ILexeme>();
 }
+

diff --git a/lexer.h b/lexer.h
index bffb28b..e3e576e 100755 (executable)
--- a/lexer.h
+++ b/lexer.h
@@ -4,12 +4,12 @@
 #include "smart_ptr.h"
 #include "utf8.h"
 #include "utf8_string.h"
+#include "lexeme.h"
 
 namespace utakata {
 
     namespace lexer {
 
-        class CLexeme;
         class CLexer
         {
             /**
@@ -33,39 +33,10 @@ namespace utakata {
                渡されたUTF8を解釈するstreamから、データを解釈して、結果を返す。
                結果は、smart_ptr<CLexeme>で返される。
             */
-            smart_ptr<CLexeme> lex(smart_ptr<utakata::utf8::CUTF8InputStream>& stream);
+            smart_ptr<utakata::lexeme::ILexeme> lex(smart_ptr<utakata::utf8::CUTF8InputStream>& stream);
 
-        private:
-        
         };
 
-        class CLexeme
-        {
-            // 非終端記号、及び終端記号を表すクラス。
-            // それぞれ分割しても問題なかったのだが、それぞれのデータが
-            // 互いに非可換であるが、それぞれ継承してもどうしようもないため、
-            // さしあたって必要となる全ての型を取得できるようにしておき、
-            // それをそのまま取得してもらう、という形にする。
-            // つまりは全体をpimplにしておき、データはそれらから取得するようにするということ。
-            // それぞれを取得するためのインターフェースはこれから作成される。
-        public:
-            CLexeme();
-            virtual ~CLexeme(){}
-
-            // 終端記号、非終端記号のIDを取得する。
-            int getID() const;
-
-            // stringのデータ型において、文字列を取得する。
-            smart_ptr<utakata::utf8_string::CUTF8String> getString() const;
-
-        private:
-            // 必要な型全てをまとめるための構造体。ただし、
-            // 内部のどれか一つだけが有効となっており、これを必要と
-            // しない終端記号のデータも存在する。
-            // 詳しくは各データを解釈する場所にて。
-            struct PImpl;
-            smart_ptr<PImpl> pimpl_;
-        };
 
     };
 

diff --git a/sublexer.cpp b/sublexer.cpp
new file mode 100644 (file)
index 0000000..63b76de
--- /dev/null
+++ b/sublexer.cpp
@@ -0,0 +1,68 @@
+#include <iostream>
+#include <assert.h>
+
+/////#include "lexeme.h"
+#include "utf8_string.h"
+#include "utf8.h"
+
+#include "sublexer.h"
+
+using namespace utakata;
+
+smart_ptr<sublexer::ISubLexer> sublexer::CFirstLex::lex(const utf8_string::CUTF8Char& ch,
+                                                        const utf8_string::CUTF8String& str,
+                                                        smart_ptr<utf8::CUTF8InputStream>& stream)
+{
+    // 最初の一文字を利用して判定を行う。
+    // なお、最初の一文字だけを行うため、strに文字が入っているとエラーになる。
+
+    assert(str.size() != 0 && "一文字目ではない。別の場所で呼びだされている");
+
+    smart_ptr<sublexer::ISubLexer> ret;
+    // ()[]`',.は構文解析の重要な要素となるため、このまま返す。
+//     if (ch.toUTF16Code() == '(' || ch.toUTF16Code() == '[')
+//     {
+//         lexeme_ = lexer::makeStartParen();
+//     }
+
+//     if (ch.toUTF16Code() == ')' || ch.toUTF16Code() == ']')
+//     {
+//         lexeme_ = lexer::makeEndParen();
+//     }
+
+//     if (ch.toUTF16Code() == '`')
+//     {
+//         lexeme_ = (lexer::makeBackQuote();
+//     }
+
+//     if (ch.toUTF16Code() == '\'')
+//     {
+//         lexeme_ = (lexer::makeQuote();
+//     }
+
+//     if (ch.toUTF16Code() == '.')
+//     {
+//         lexeme_ = lexer::makeDot();
+//     }
+
+//     if (ch.toUTF16Code() == ',' || ch.toUTF16Code() == '#')
+//     {
+//         // ,や#の場合、次に別の文字が続く可能性があるため、NextLexemeLexerを
+//         // 呼び出すようにしておく。
+//         ret.add(new lexer::CNextLexemeLexer());
+//     }
+
+//     if (ch.toUTF16Code() == '"')
+//     {
+//         // 先頭が"の場合、stringと判断される。
+//         ret.add(new lexer::CStringLexer());
+//     }
+
+    // 判定できないとき、
+    return ret;
+}
+
+smart_ptr<lexeme::ILexeme> sublexer::CFirstLex::getLexeme()
+{
+    return smart_ptr<lexeme::ILexeme>();
+}

diff --git a/sublexer.h b/sublexer.h
new file mode 100644 (file)
index 0000000..4851ad4
--- /dev/null
+++ b/sublexer.h
@@ -0,0 +1,67 @@
+#ifndef _SUBLEXER_H_
+#define _SUBLEXER_H_
+
+#include "smart_ptr.h"
+#include "utf8.h"
+#include "utf8_string.h"
+#include "lexeme.h"
+
+namespace utakata {
+
+    namespace sublexer {
+
+        class ISubLexer
+        {
+            /**
+               lexerのlex内で、数々の条件分岐を行わなければならない場合に、
+               それぞれのsublexerからの結果、追加で返されるsublexerを、
+               次の文字における判定として利用することにしておく。
+               このようにすることで、インターフェースを完全に統一しながら、
+               sublexerからsublexerを返すようにしていくだけで、
+               lex内部をシンプルにすることができる。
+
+               これは完全にインターフェースのみを提供する。
+            */
+               
+        public:
+            ISubLexer() {}
+            virtual ~ISubLexer() {}
+
+            // 処理の実体となる。
+            // strは前回のsublexが返した文字列であり、前のsublexによって更新される。
+            // streamは文字通りストリームとなる。sublex中で文字の取得などを
+            // 自由に行う必要があるために渡している。
+            virtual smart_ptr<ISubLexer> lex(const utakata::utf8_string::CUTF8Char& ch,
+                                             const utakata::utf8_string::CUTF8String& str,
+                                             smart_ptr<utakata::utf8::CUTF8InputStream>& stream) = 0;
+          
+            // 作成したILexemeを返す。lexの結果を返さない場合に利用される。
+            // これだけは前方参照のみでなんとか乗り切る必要がある。
+            virtual smart_ptr<utakata::lexeme::ILexeme> getLexeme() = 0;
+
+        };
+
+        class CFirstLex : public ISubLexer
+        {
+            // lex関数の内部で最初に実行される。
+            // ここから返されない場合は、一文字の構文構成文字が返されるということになる。
+        public:
+
+            smart_ptr<ISubLexer> lex(const utakata::utf8_string::CUTF8Char& ch,
+                                     const utakata::utf8_string::CUTF8String& str,
+                                     smart_ptr<utakata::utf8::CUTF8InputStream>& stream);
+
+            smart_ptr<utakata::lexeme::ILexeme> getLexeme();
+
+        private:
+            // 返すべきILexemeが確定した場合に、ここに格納する。
+            smart_ptr<utakata::lexeme::ILexeme> lexeme_;
+        };
+
+
+
+    };
+
+};
+
+#endif /* _SUBLEXER_H_ */

diff --git a/test/GPATH b/test/GPATH
index 4b8dbcd..be3f2f2 100644 (file)
Binary files a/test/GPATH and b/test/GPATH differ

diff --git a/test/GRTAGS b/test/GRTAGS
index 87bc221..833eb7e 100644 (file)
Binary files a/test/GRTAGS and b/test/GRTAGS differ

diff --git a/test/GSYMS b/test/GSYMS
index e09ffd6..4c4c827 100644 (file)
Binary files a/test/GSYMS and b/test/GSYMS differ

diff --git a/test/GTAGS b/test/GTAGS
index 59695a7..7248ca0 100644 (file)
Binary files a/test/GTAGS and b/test/GTAGS differ

diff --git a/test/Makefile b/test/Makefile
index ca4ee04..448f1d9 100644 (file)
--- a/test/Makefile
+++ b/test/Makefile
@@ -31,7 +31,8 @@ NORMAL_UNINSTALL = :
 PRE_UNINSTALL = :
 POST_UNINSTALL = :
 bin_PROGRAMS = utf8_test$(EXEEXT) textarrayformat_test$(EXEEXT) \
-       utf8_string_test$(EXEEXT)
+       utf8_string_test$(EXEEXT) lexer_test$(EXEEXT) \
+       sublexer_test$(EXEEXT)
 subdir = test
 DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
@@ -44,6 +45,15 @@ CONFIG_CLEAN_FILES =
 am__installdirs = "$(DESTDIR)$(bindir)"
 binPROGRAMS_INSTALL = $(INSTALL_PROGRAM)
 PROGRAMS = $(bin_PROGRAMS)
+am_lexer_test_OBJECTS = lexer_test.$(OBJEXT) lexer.$(OBJEXT) \
+       sublexer.$(OBJEXT) simpletest.$(OBJEXT) utf8_string.$(OBJEXT) \
+       utf8.$(OBJEXT)
+lexer_test_OBJECTS = $(am_lexer_test_OBJECTS)
+lexer_test_LDADD = $(LDADD)
+am_sublexer_test_OBJECTS = sublexer_test.$(OBJEXT) sublexer.$(OBJEXT) \
+       simpletest.$(OBJEXT) utf8_string.$(OBJEXT) utf8.$(OBJEXT)
+sublexer_test_OBJECTS = $(am_sublexer_test_OBJECTS)
+sublexer_test_LDADD = $(LDADD)
 am_textarrayformat_test_OBJECTS = textarrayformat_test.$(OBJEXT) \
        TextArrayFormat.$(OBJEXT) simpletest.$(OBJEXT)
 textarrayformat_test_OBJECTS = $(am_textarrayformat_test_OBJECTS)
@@ -64,10 +74,12 @@ CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
 CXXLD = $(CXX)
 CXXLINK = $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \
        -o $@
-SOURCES = $(textarrayformat_test_SOURCES) $(utf8_string_test_SOURCES) \
+SOURCES = $(lexer_test_SOURCES) $(sublexer_test_SOURCES) \
+       $(textarrayformat_test_SOURCES) $(utf8_string_test_SOURCES) \
+       $(utf8_test_SOURCES)
+DIST_SOURCES = $(lexer_test_SOURCES) $(sublexer_test_SOURCES) \
+       $(textarrayformat_test_SOURCES) $(utf8_string_test_SOURCES) \
        $(utf8_test_SOURCES)
-DIST_SOURCES = $(textarrayformat_test_SOURCES) \
-       $(utf8_string_test_SOURCES) $(utf8_test_SOURCES)
 ETAGS = etags
 CTAGS = ctags
 DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
@@ -81,7 +93,7 @@ CC = gcc
 CCDEPMODE = depmode=gcc3
 CFLAGS = -g -O2
 CPP = gcc -E
-CPPFLAGS = -Wall
+CPPFLAGS = 
 CXX = g++
 CXXDEPMODE = depmode=gcc3
 CXXFLAGS = -g -O2
@@ -160,10 +172,15 @@ target_alias =
 top_build_prefix = ../
 top_builddir = ..
 top_srcdir = ..
+
+# 警告レベルを最大にする。
+AM_CPPFLAGS = -Wall
 INCLUDES = -l$(top_srcdir)
 utf8_string_test_SOURCES = utf8_string_test.cpp ../utf8_string.cpp ../simpletest.cpp ../utf8.cpp
 utf8_test_SOURCES = utf8_test.cpp ../utf8.cpp ../simpletest.cpp
 textarrayformat_test_SOURCES = textarrayformat_test.cpp ../TextArrayFormat.cpp ../simpletest.cpp
+lexer_test_SOURCES = lexer_test.cpp ../lexer.cpp ../sublexer.cpp ../simpletest.cpp ../utf8_string.cpp ../utf8.cpp
+sublexer_test_SOURCES = sublexer_test.cpp ../sublexer.cpp ../simpletest.cpp ../utf8_string.cpp ../utf8.cpp
 check_PROGRANS = $(bin_PROGRAMS)
 TESTS = $(bin_PROGRAMS)
 all: all-am
@@ -222,6 +239,12 @@ uninstall-binPROGRAMS:
 
 clean-binPROGRAMS:
        -test -z "$(bin_PROGRAMS)" || rm -f $(bin_PROGRAMS)
+lexer_test$(EXEEXT): $(lexer_test_OBJECTS) $(lexer_test_DEPENDENCIES) 
+       @rm -f lexer_test$(EXEEXT)
+       $(CXXLINK) $(lexer_test_OBJECTS) $(lexer_test_LDADD) $(LIBS)
+sublexer_test$(EXEEXT): $(sublexer_test_OBJECTS) $(sublexer_test_DEPENDENCIES) 
+       @rm -f sublexer_test$(EXEEXT)
+       $(CXXLINK) $(sublexer_test_OBJECTS) $(sublexer_test_LDADD) $(LIBS)
 textarrayformat_test$(EXEEXT): $(textarrayformat_test_OBJECTS) $(textarrayformat_test_DEPENDENCIES) 
        @rm -f textarrayformat_test$(EXEEXT)
        $(CXXLINK) $(textarrayformat_test_OBJECTS) $(textarrayformat_test_LDADD) $(LIBS)
@@ -239,7 +262,11 @@ distclean-compile:
        -rm -f *.tab.c
 
 include ./$(DEPDIR)/TextArrayFormat.Po
+include ./$(DEPDIR)/lexer.Po
+include ./$(DEPDIR)/lexer_test.Po
 include ./$(DEPDIR)/simpletest.Po
+include ./$(DEPDIR)/sublexer.Po
+include ./$(DEPDIR)/sublexer_test.Po
 include ./$(DEPDIR)/textarrayformat_test.Po
 include ./$(DEPDIR)/utf8.Po
 include ./$(DEPDIR)/utf8_string.Po
@@ -260,19 +287,33 @@ include ./$(DEPDIR)/utf8_test.Po
 #      DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) \
 #      $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
 
-TextArrayFormat.o: ../TextArrayFormat.cpp
-       $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT TextArrayFormat.o -MD -MP -MF $(DEPDIR)/TextArrayFormat.Tpo -c -o TextArrayFormat.o `test -f '../TextArrayFormat.cpp' || echo '$(srcdir)/'`../TextArrayFormat.cpp
-       mv -f $(DEPDIR)/TextArrayFormat.Tpo $(DEPDIR)/TextArrayFormat.Po
-#      source='../TextArrayFormat.cpp' object='TextArrayFormat.o' libtool=no \
+lexer.o: ../lexer.cpp
+       $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT lexer.o -MD -MP -MF $(DEPDIR)/lexer.Tpo -c -o lexer.o `test -f '../lexer.cpp' || echo '$(srcdir)/'`../lexer.cpp
+       mv -f $(DEPDIR)/lexer.Tpo $(DEPDIR)/lexer.Po
+#      source='../lexer.cpp' object='lexer.o' libtool=no \
 #      DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) \
-#      $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o TextArrayFormat.o `test -f '../TextArrayFormat.cpp' || echo '$(srcdir)/'`../TextArrayFormat.cpp
+#      $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o lexer.o `test -f '../lexer.cpp' || echo '$(srcdir)/'`../lexer.cpp
 
-TextArrayFormat.obj: ../TextArrayFormat.cpp
-       $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT TextArrayFormat.obj -MD -MP -MF $(DEPDIR)/TextArrayFormat.Tpo -c -o TextArrayFormat.obj `if test -f '../TextArrayFormat.cpp'; then $(CYGPATH_W) '../TextArrayFormat.cpp'; else $(CYGPATH_W) '$(srcdir)/../TextArrayFormat.cpp'; fi`
-       mv -f $(DEPDIR)/TextArrayFormat.Tpo $(DEPDIR)/TextArrayFormat.Po
-#      source='../TextArrayFormat.cpp' object='TextArrayFormat.obj' libtool=no \
+lexer.obj: ../lexer.cpp
+       $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT lexer.obj -MD -MP -MF $(DEPDIR)/lexer.Tpo -c -o lexer.obj `if test -f '../lexer.cpp'; then $(CYGPATH_W) '../lexer.cpp'; else $(CYGPATH_W) '$(srcdir)/../lexer.cpp'; fi`
+       mv -f $(DEPDIR)/lexer.Tpo $(DEPDIR)/lexer.Po
+#      source='../lexer.cpp' object='lexer.obj' libtool=no \
 #      DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) \
-#      $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o TextArrayFormat.obj `if test -f '../TextArrayFormat.cpp'; then $(CYGPATH_W) '../TextArrayFormat.cpp'; else $(CYGPATH_W) '$(srcdir)/../TextArrayFormat.cpp'; fi`
+#      $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o lexer.obj `if test -f '../lexer.cpp'; then $(CYGPATH_W) '../lexer.cpp'; else $(CYGPATH_W) '$(srcdir)/../lexer.cpp'; fi`
+
+sublexer.o: ../sublexer.cpp
+       $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT sublexer.o -MD -MP -MF $(DEPDIR)/sublexer.Tpo -c -o sublexer.o `test -f '../sublexer.cpp' || echo '$(srcdir)/'`../sublexer.cpp
+       mv -f $(DEPDIR)/sublexer.Tpo $(DEPDIR)/sublexer.Po
+#      source='../sublexer.cpp' object='sublexer.o' libtool=no \
+#      DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) \
+#      $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o sublexer.o `test -f '../sublexer.cpp' || echo '$(srcdir)/'`../sublexer.cpp
+
+sublexer.obj: ../sublexer.cpp
+       $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT sublexer.obj -MD -MP -MF $(DEPDIR)/sublexer.Tpo -c -o sublexer.obj `if test -f '../sublexer.cpp'; then $(CYGPATH_W) '../sublexer.cpp'; else $(CYGPATH_W) '$(srcdir)/../sublexer.cpp'; fi`
+       mv -f $(DEPDIR)/sublexer.Tpo $(DEPDIR)/sublexer.Po
+#      source='../sublexer.cpp' object='sublexer.obj' libtool=no \
+#      DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) \
+#      $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o sublexer.obj `if test -f '../sublexer.cpp'; then $(CYGPATH_W) '../sublexer.cpp'; else $(CYGPATH_W) '$(srcdir)/../sublexer.cpp'; fi`
 
 simpletest.o: ../simpletest.cpp
        $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT simpletest.o -MD -MP -MF $(DEPDIR)/simpletest.Tpo -c -o simpletest.o `test -f '../simpletest.cpp' || echo '$(srcdir)/'`../simpletest.cpp
@@ -316,6 +357,20 @@ utf8.obj: ../utf8.cpp
 #      DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) \
 #      $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o utf8.obj `if test -f '../utf8.cpp'; then $(CYGPATH_W) '../utf8.cpp'; else $(CYGPATH_W) '$(srcdir)/../utf8.cpp'; fi`
 
+TextArrayFormat.o: ../TextArrayFormat.cpp
+       $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT TextArrayFormat.o -MD -MP -MF $(DEPDIR)/TextArrayFormat.Tpo -c -o TextArrayFormat.o `test -f '../TextArrayFormat.cpp' || echo '$(srcdir)/'`../TextArrayFormat.cpp
+       mv -f $(DEPDIR)/TextArrayFormat.Tpo $(DEPDIR)/TextArrayFormat.Po
+#      source='../TextArrayFormat.cpp' object='TextArrayFormat.o' libtool=no \
+#      DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) \
+#      $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o TextArrayFormat.o `test -f '../TextArrayFormat.cpp' || echo '$(srcdir)/'`../TextArrayFormat.cpp
+
+TextArrayFormat.obj: ../TextArrayFormat.cpp
+       $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT TextArrayFormat.obj -MD -MP -MF $(DEPDIR)/TextArrayFormat.Tpo -c -o TextArrayFormat.obj `if test -f '../TextArrayFormat.cpp'; then $(CYGPATH_W) '../TextArrayFormat.cpp'; else $(CYGPATH_W) '$(srcdir)/../TextArrayFormat.cpp'; fi`
+       mv -f $(DEPDIR)/TextArrayFormat.Tpo $(DEPDIR)/TextArrayFormat.Po
+#      source='../TextArrayFormat.cpp' object='TextArrayFormat.obj' libtool=no \
+#      DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) \
+#      $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o TextArrayFormat.obj `if test -f '../TextArrayFormat.cpp'; then $(CYGPATH_W) '../TextArrayFormat.cpp'; else $(CYGPATH_W) '$(srcdir)/../TextArrayFormat.cpp'; fi`
+
 ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
        list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
        unique=`for i in $$list; do \

diff --git a/test/Makefile.am b/test/Makefile.am
index 9faefa4..af88302 100755 (executable)
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -1,10 +1,14 @@
+# 警告レベルを最大にする。
+AM_CPPFLAGS = -Wall
 INCLUDES = -l$(top_srcdir)
 
-bin_PROGRAMS = utf8_test textarrayformat_test utf8_string_test
+bin_PROGRAMS = utf8_test textarrayformat_test utf8_string_test lexer_test sublexer_test
 
 utf8_string_test_SOURCES = utf8_string_test.cpp ../utf8_string.cpp ../simpletest.cpp ../utf8.cpp
 utf8_test_SOURCES = utf8_test.cpp ../utf8.cpp ../simpletest.cpp
 textarrayformat_test_SOURCES = textarrayformat_test.cpp ../TextArrayFormat.cpp ../simpletest.cpp
+lexer_test_SOURCES = lexer_test.cpp ../lexer.cpp ../sublexer.cpp ../simpletest.cpp ../utf8_string.cpp ../utf8.cpp
+sublexer_test_SOURCES = sublexer_test.cpp ../sublexer.cpp ../simpletest.cpp ../utf8_string.cpp ../utf8.cpp
 
 check_PROGRANS = $(bin_PROGRAMS)
 TESTS = $(bin_PROGRAMS)

diff --git a/test/Makefile.in b/test/Makefile.in
index 59aec5e..a1ee06c 100644 (file)
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -31,7 +31,8 @@ NORMAL_UNINSTALL = :
 PRE_UNINSTALL = :
 POST_UNINSTALL = :
 bin_PROGRAMS = utf8_test$(EXEEXT) textarrayformat_test$(EXEEXT) \
-       utf8_string_test$(EXEEXT)
+       utf8_string_test$(EXEEXT) lexer_test$(EXEEXT) \
+       sublexer_test$(EXEEXT)
 subdir = test
 DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
@@ -44,6 +45,15 @@ CONFIG_CLEAN_FILES =
 am__installdirs = "$(DESTDIR)$(bindir)"
 binPROGRAMS_INSTALL = $(INSTALL_PROGRAM)
 PROGRAMS = $(bin_PROGRAMS)
+am_lexer_test_OBJECTS = lexer_test.$(OBJEXT) lexer.$(OBJEXT) \
+       sublexer.$(OBJEXT) simpletest.$(OBJEXT) utf8_string.$(OBJEXT) \
+       utf8.$(OBJEXT)
+lexer_test_OBJECTS = $(am_lexer_test_OBJECTS)
+lexer_test_LDADD = $(LDADD)
+am_sublexer_test_OBJECTS = sublexer_test.$(OBJEXT) sublexer.$(OBJEXT) \
+       simpletest.$(OBJEXT) utf8_string.$(OBJEXT) utf8.$(OBJEXT)
+sublexer_test_OBJECTS = $(am_sublexer_test_OBJECTS)
+sublexer_test_LDADD = $(LDADD)
 am_textarrayformat_test_OBJECTS = textarrayformat_test.$(OBJEXT) \
        TextArrayFormat.$(OBJEXT) simpletest.$(OBJEXT)
 textarrayformat_test_OBJECTS = $(am_textarrayformat_test_OBJECTS)
@@ -64,10 +74,12 @@ CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
 CXXLD = $(CXX)
 CXXLINK = $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \
        -o $@
-SOURCES = $(textarrayformat_test_SOURCES) $(utf8_string_test_SOURCES) \
+SOURCES = $(lexer_test_SOURCES) $(sublexer_test_SOURCES) \
+       $(textarrayformat_test_SOURCES) $(utf8_string_test_SOURCES) \
+       $(utf8_test_SOURCES)
+DIST_SOURCES = $(lexer_test_SOURCES) $(sublexer_test_SOURCES) \
+       $(textarrayformat_test_SOURCES) $(utf8_string_test_SOURCES) \
        $(utf8_test_SOURCES)
-DIST_SOURCES = $(textarrayformat_test_SOURCES) \
-       $(utf8_string_test_SOURCES) $(utf8_test_SOURCES)
 ETAGS = etags
 CTAGS = ctags
 DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
@@ -160,10 +172,15 @@ target_alias = @target_alias@
 top_build_prefix = @top_build_prefix@
 top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
+
+# 警告レベルを最大にする。
+AM_CPPFLAGS = -Wall
 INCLUDES = -l$(top_srcdir)
 utf8_string_test_SOURCES = utf8_string_test.cpp ../utf8_string.cpp ../simpletest.cpp ../utf8.cpp
 utf8_test_SOURCES = utf8_test.cpp ../utf8.cpp ../simpletest.cpp
 textarrayformat_test_SOURCES = textarrayformat_test.cpp ../TextArrayFormat.cpp ../simpletest.cpp
+lexer_test_SOURCES = lexer_test.cpp ../lexer.cpp ../sublexer.cpp ../simpletest.cpp ../utf8_string.cpp ../utf8.cpp
+sublexer_test_SOURCES = sublexer_test.cpp ../sublexer.cpp ../simpletest.cpp ../utf8_string.cpp ../utf8.cpp
 check_PROGRANS = $(bin_PROGRAMS)
 TESTS = $(bin_PROGRAMS)
 all: all-am
@@ -222,6 +239,12 @@ uninstall-binPROGRAMS:
 
 clean-binPROGRAMS:
        -test -z "$(bin_PROGRAMS)" || rm -f $(bin_PROGRAMS)
+lexer_test$(EXEEXT): $(lexer_test_OBJECTS) $(lexer_test_DEPENDENCIES) 
+       @rm -f lexer_test$(EXEEXT)
+       $(CXXLINK) $(lexer_test_OBJECTS) $(lexer_test_LDADD) $(LIBS)
+sublexer_test$(EXEEXT): $(sublexer_test_OBJECTS) $(sublexer_test_DEPENDENCIES) 
+       @rm -f sublexer_test$(EXEEXT)
+       $(CXXLINK) $(sublexer_test_OBJECTS) $(sublexer_test_LDADD) $(LIBS)
 textarrayformat_test$(EXEEXT): $(textarrayformat_test_OBJECTS) $(textarrayformat_test_DEPENDENCIES) 
        @rm -f textarrayformat_test$(EXEEXT)
        $(CXXLINK) $(textarrayformat_test_OBJECTS) $(textarrayformat_test_LDADD) $(LIBS)
@@ -239,7 +262,11 @@ distclean-compile:
        -rm -f *.tab.c
 
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TextArrayFormat.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lexer.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lexer_test.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/simpletest.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sublexer.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sublexer_test.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/textarrayformat_test.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/utf8.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/utf8_string.Po@am__quote@
@@ -260,19 +287,33 @@ distclean-compile:
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
 @am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
 
-TextArrayFormat.o: ../TextArrayFormat.cpp
-@am__fastdepCXX_TRUE@  $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT TextArrayFormat.o -MD -MP -MF $(DEPDIR)/TextArrayFormat.Tpo -c -o TextArrayFormat.o `test -f '../TextArrayFormat.cpp' || echo '$(srcdir)/'`../TextArrayFormat.cpp
-@am__fastdepCXX_TRUE@  mv -f $(DEPDIR)/TextArrayFormat.Tpo $(DEPDIR)/TextArrayFormat.Po
-@AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='../TextArrayFormat.cpp' object='TextArrayFormat.o' libtool=no @AMDEPBACKSLASH@
+lexer.o: ../lexer.cpp
+@am__fastdepCXX_TRUE@  $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT lexer.o -MD -MP -MF $(DEPDIR)/lexer.Tpo -c -o lexer.o `test -f '../lexer.cpp' || echo '$(srcdir)/'`../lexer.cpp
+@am__fastdepCXX_TRUE@  mv -f $(DEPDIR)/lexer.Tpo $(DEPDIR)/lexer.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='../lexer.cpp' object='lexer.o' libtool=no @AMDEPBACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o TextArrayFormat.o `test -f '../TextArrayFormat.cpp' || echo '$(srcdir)/'`../TextArrayFormat.cpp
+@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o lexer.o `test -f '../lexer.cpp' || echo '$(srcdir)/'`../lexer.cpp
 
-TextArrayFormat.obj: ../TextArrayFormat.cpp
-@am__fastdepCXX_TRUE@  $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT TextArrayFormat.obj -MD -MP -MF $(DEPDIR)/TextArrayFormat.Tpo -c -o TextArrayFormat.obj `if test -f '../TextArrayFormat.cpp'; then $(CYGPATH_W) '../TextArrayFormat.cpp'; else $(CYGPATH_W) '$(srcdir)/../TextArrayFormat.cpp'; fi`
-@am__fastdepCXX_TRUE@  mv -f $(DEPDIR)/TextArrayFormat.Tpo $(DEPDIR)/TextArrayFormat.Po
-@AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='../TextArrayFormat.cpp' object='TextArrayFormat.obj' libtool=no @AMDEPBACKSLASH@
+lexer.obj: ../lexer.cpp
+@am__fastdepCXX_TRUE@  $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT lexer.obj -MD -MP -MF $(DEPDIR)/lexer.Tpo -c -o lexer.obj `if test -f '../lexer.cpp'; then $(CYGPATH_W) '../lexer.cpp'; else $(CYGPATH_W) '$(srcdir)/../lexer.cpp'; fi`
+@am__fastdepCXX_TRUE@  mv -f $(DEPDIR)/lexer.Tpo $(DEPDIR)/lexer.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='../lexer.cpp' object='lexer.obj' libtool=no @AMDEPBACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o TextArrayFormat.obj `if test -f '../TextArrayFormat.cpp'; then $(CYGPATH_W) '../TextArrayFormat.cpp'; else $(CYGPATH_W) '$(srcdir)/../TextArrayFormat.cpp'; fi`
+@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o lexer.obj `if test -f '../lexer.cpp'; then $(CYGPATH_W) '../lexer.cpp'; else $(CYGPATH_W) '$(srcdir)/../lexer.cpp'; fi`
+
+sublexer.o: ../sublexer.cpp
+@am__fastdepCXX_TRUE@  $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT sublexer.o -MD -MP -MF $(DEPDIR)/sublexer.Tpo -c -o sublexer.o `test -f '../sublexer.cpp' || echo '$(srcdir)/'`../sublexer.cpp
+@am__fastdepCXX_TRUE@  mv -f $(DEPDIR)/sublexer.Tpo $(DEPDIR)/sublexer.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='../sublexer.cpp' object='sublexer.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o sublexer.o `test -f '../sublexer.cpp' || echo '$(srcdir)/'`../sublexer.cpp
+
+sublexer.obj: ../sublexer.cpp
+@am__fastdepCXX_TRUE@  $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT sublexer.obj -MD -MP -MF $(DEPDIR)/sublexer.Tpo -c -o sublexer.obj `if test -f '../sublexer.cpp'; then $(CYGPATH_W) '../sublexer.cpp'; else $(CYGPATH_W) '$(srcdir)/../sublexer.cpp'; fi`
+@am__fastdepCXX_TRUE@  mv -f $(DEPDIR)/sublexer.Tpo $(DEPDIR)/sublexer.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='../sublexer.cpp' object='sublexer.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o sublexer.obj `if test -f '../sublexer.cpp'; then $(CYGPATH_W) '../sublexer.cpp'; else $(CYGPATH_W) '$(srcdir)/../sublexer.cpp'; fi`
 
 simpletest.o: ../simpletest.cpp
 @am__fastdepCXX_TRUE@  $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT simpletest.o -MD -MP -MF $(DEPDIR)/simpletest.Tpo -c -o simpletest.o `test -f '../simpletest.cpp' || echo '$(srcdir)/'`../simpletest.cpp
@@ -316,6 +357,20 @@ utf8.obj: ../utf8.cpp
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
 @am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o utf8.obj `if test -f '../utf8.cpp'; then $(CYGPATH_W) '../utf8.cpp'; else $(CYGPATH_W) '$(srcdir)/../utf8.cpp'; fi`
 
+TextArrayFormat.o: ../TextArrayFormat.cpp
+@am__fastdepCXX_TRUE@  $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT TextArrayFormat.o -MD -MP -MF $(DEPDIR)/TextArrayFormat.Tpo -c -o TextArrayFormat.o `test -f '../TextArrayFormat.cpp' || echo '$(srcdir)/'`../TextArrayFormat.cpp
+@am__fastdepCXX_TRUE@  mv -f $(DEPDIR)/TextArrayFormat.Tpo $(DEPDIR)/TextArrayFormat.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='../TextArrayFormat.cpp' object='TextArrayFormat.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o TextArrayFormat.o `test -f '../TextArrayFormat.cpp' || echo '$(srcdir)/'`../TextArrayFormat.cpp
+
+TextArrayFormat.obj: ../TextArrayFormat.cpp
+@am__fastdepCXX_TRUE@  $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT TextArrayFormat.obj -MD -MP -MF $(DEPDIR)/TextArrayFormat.Tpo -c -o TextArrayFormat.obj `if test -f '../TextArrayFormat.cpp'; then $(CYGPATH_W) '../TextArrayFormat.cpp'; else $(CYGPATH_W) '$(srcdir)/../TextArrayFormat.cpp'; fi`
+@am__fastdepCXX_TRUE@  mv -f $(DEPDIR)/TextArrayFormat.Tpo $(DEPDIR)/TextArrayFormat.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='../TextArrayFormat.cpp' object='TextArrayFormat.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o TextArrayFormat.obj `if test -f '../TextArrayFormat.cpp'; then $(CYGPATH_W) '../TextArrayFormat.cpp'; else $(CYGPATH_W) '$(srcdir)/../TextArrayFormat.cpp'; fi`
+
 ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
        list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
        unique=`for i in $$list; do \

diff --git a/test/lexer_test.cpp b/test/lexer_test.cpp
new file mode 100644 (file)
index 0000000..4532f48
--- /dev/null
+++ b/test/lexer_test.cpp
@@ -0,0 +1,20 @@
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <functional>
+
+
+#include "../simpletest.h"
+
+#include "../lexer.h"
+
+
+int main(int argc, char *argv[])
+{
+    simpletest::CSimpleTestSuite suite("main lexer test");
+//     suite.addTester(sfcr::screate(utf8_multichar_test, suite.getAsserter()));
+//     suite.addTester(sfcr::screate(utf8_string_test, suite.getAsserter()));
+//     suite.addTester(sfcr::screate(utf8_string_util_test, suite.getAsserter()));
+    suite.run();
+    return 0;
+}

diff --git a/test/sublexer_test.cpp b/test/sublexer_test.cpp
new file mode 100644 (file)
index 0000000..98dbd73
--- /dev/null
+++ b/test/sublexer_test.cpp
@@ -0,0 +1,20 @@
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <functional>
+
+
+#include "../simpletest.h"
+
+#include "../sublexer.h"
+
+
+int main(int argc, char *argv[])
+{
+    simpletest::CSimpleTestSuite suite("sublexer's test");
+//     suite.addTester(sfcr::screate(utf8_multichar_test, suite.getAsserter()));
+//     suite.addTester(sfcr::screate(utf8_string_test, suite.getAsserter()));
+//     suite.addTester(sfcr::screate(utf8_string_util_test, suite.getAsserter()));
+    suite.run();
+    return 0;
+}

diff --git a/test/utf8_string_test b/test/utf8_string_test
index 450afc0..cd41954 100755 (executable)
Binary files a/test/utf8_string_test and b/test/utf8_string_test differ

diff --git a/test/utf8_string_test.cpp b/test/utf8_string_test.cpp
index d8680e1..43f4bca 100644 (file)
--- a/test/utf8_string_test.cpp
+++ b/test/utf8_string_test.cpp
@@ -23,12 +23,12 @@ bool utf8_multichar_test(smart_ptr<simpletest::CSimpleTestAsserter> asserter)
 
     // 単独のreadを試す。
     utakata::utf8_string::CUTF8Char ch(stream.read());
-    asserter->check(ch.toUTF16Code(), utakata::utf8::generateUTF8Code("あ"));
+    asserter->check(ch.toUTF16Code(), utakata::utf8::generateUTF8Code("あ"), "通常read失敗");
     // peekが問題なく働いていることのテスト
     utakata::utf8_string::CUTF8Char ch2(stream.peek());
-    asserter->check(ch2.toUTF16Code(), utakata::utf8::generateUTF8Code("い"));
+    asserter->check(ch2.toUTF16Code(), utakata::utf8::generateUTF8Code("い"), "peek失敗");
     utakata::utf8_string::CUTF8Char ch3(stream.read());
-    asserter->check(ch3.toUTF16Code(), utakata::utf8::generateUTF8Code("い"));
+    asserter->check(ch3.toUTF16Code(), utakata::utf8::generateUTF8Code("い"), "peek後read失敗");
 
     // asciiだとしても問題無く読みだせるはず。
     utakata::utf8_string::CUTF8Char ch4(stream.peek());
@@ -76,7 +76,7 @@ bool utf8_string_test(smart_ptr<simpletest::CSimpleTestAsserter> asserter)
 
     // 文字との加算も可能。
     str += ch;
-    asserter->check(str.toStr(), "ã\81\82ã\81\82ã\81\84ã\81\86ã\81\88sã\81\82ã\81\84ã\81\86ã\81\88s");
+    asserter->check(str.toStr(), "ã\81\82ã\81\84ã\81\86ã\81\88sã\81\82ã\81\84ã\81\86ã\81\88sã\81\82");
 
     return asserter->isOk();
 }
@@ -107,7 +107,7 @@ bool utf8_string_util_test(smart_ptr<simpletest::CSimpleTestAsserter> asserter)
 
     // 互いに加算できる。
     utakata::utf8_string::CUTF8String str4 = str + str2;
-    asserter->check(str4.toStr(), "ssおssおあいうえs");
+    asserter->check(str4.toStr(), "ssおあいうえsssお");
 
     return asserter->isOk();
 }