fix uncorrect parsing when anno->len > 1

diff --git a/lib/tokenizer.c b/lib/tokenizer.c
index 58b7a3b..824548c 100644 (file)
--- a/lib/tokenizer.c
+++ b/lib/tokenizer.c
@@ -27,7 +27,7 @@
  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * $Id: tokenizer.c,v 1.2 2007/03/25 11:01:15 kazuma-t Exp $
+ * $Id: tokenizer.c,v 1.3 2007/03/25 16:25:46 kazuma-t Exp $
  */
 
 #include <string.h>
@@ -150,7 +150,7 @@ cha_tok_parse(chasen_tok_t *tok, unsigned char *str, char *type, int len,
     if (anno_no != NULL && (no = is_anno(tok, str, len)) < 0) {
        anno = &(tok->anno[-no]);
        *anno_no = -no;
-       for (cursor = tok->mblen(str, len);
+       for (cursor = anno->len1;
             cursor < len;
             cursor += tok->mblen(str + cursor, len - cursor)) {
            if (is_anno2(anno, str, cursor))