first step of using newtokenlib if possible

diff --git a/src/ltjsarticle.cls b/src/ltjsarticle.cls
index 12e804b..5d0e1b4 100644 (file)
--- a/src/ltjsarticle.cls
+++ b/src/ltjsarticle.cls
@@ -366,13 +366,25 @@
     \fi
   \fi
   \ltj@setpar@global
-  \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
-  \@tempskipa=\ltjgetparameter{xkanjiskip}
-  \ifdim\@tempskipa>\z@
-    \if@slide
-      \ltjsetxkanjiskip{0.1em}
-    \else
-      \ltjsetxkanjiskip{0.25em plus 0.15em minus 0.06em}
+  \ifltj@token@avail@scan
+    \ltjsetkanjiskip\z@ plus .1\zw minus .01\zw
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide
+        \ltjsetxkanjiskip .1em
+      \else
+       \ltjsetxkanjiskip .25em plus .15em minus .06em
+      \fi
+    \fi
+  \else
+    \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide
+        \ltjsetxkanjiskip{.1em}
+      \else
+        \ltjsetxkanjiskip{.25em plus .15em minus .06em}
+      \fi
     \fi
   \fi}
 \emergencystretch 3\zw

diff --git a/src/ltjsbook.cls b/src/ltjsbook.cls
index 22ef804..382b45d 100644 (file)
--- a/src/ltjsbook.cls
+++ b/src/ltjsbook.cls
@@ -373,13 +373,25 @@
     \fi
   \fi
   \ltj@setpar@global
-  \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
-  \@tempskipa=\ltjgetparameter{xkanjiskip}
-  \ifdim\@tempskipa>\z@
-    \if@slide
-      \ltjsetxkanjiskip{0.1em}
-    \else
-      \ltjsetxkanjiskip{0.25em plus 0.15em minus 0.06em}
+  \ifltj@token@avail@scan
+    \ltjsetkanjiskip\z@ plus .1\zw minus .01\zw
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide
+        \ltjsetxkanjiskip .1em
+      \else
+       \ltjsetxkanjiskip .25em plus .15em minus .06em
+      \fi
+    \fi
+  \else
+    \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide
+        \ltjsetxkanjiskip{.1em}
+      \else
+        \ltjsetxkanjiskip{.25em plus .15em minus .06em}
+      \fi
     \fi
   \fi}
 \emergencystretch 3\zw

diff --git a/src/ltjsclasses.dtx b/src/ltjsclasses.dtx
index bccb49b..238a4cb 100644 (file)
--- a/src/ltjsclasses.dtx
+++ b/src/ltjsclasses.dtx
@@ -989,13 +989,25 @@
     \fi
   \fi
   \ltj@setpar@global
-  \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
-  \@tempskipa=\ltjgetparameter{xkanjiskip}
-  \ifdim\@tempskipa>\z@
-    \if@slide 
-      \ltjsetxkanjiskip{0.1em}
-    \else
-      \ltjsetxkanjiskip{0.25em plus 0.15em minus 0.06em}
+  \ifltj@token@avail@scan
+    \ltjsetkanjiskip\z@ plus .1\zw minus .01\zw
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide 
+        \ltjsetxkanjiskip .1em
+      \else
+       \ltjsetxkanjiskip .25em plus .15em minus .06em
+      \fi
+    \fi
+  \else
+    \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide 
+        \ltjsetxkanjiskip{.1em}
+      \else
+        \ltjsetxkanjiskip{.25em plus .15em minus .06em}
+      \fi
     \fi
   \fi}
 %    \end{macrocode}

diff --git a/src/ltjskiyou.cls b/src/ltjskiyou.cls
index 45ad2b9..54ed5ca 100644 (file)
--- a/src/ltjskiyou.cls
+++ b/src/ltjskiyou.cls
@@ -368,13 +368,25 @@
     \fi
   \fi
   \ltj@setpar@global
-  \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
-  \@tempskipa=\ltjgetparameter{xkanjiskip}
-  \ifdim\@tempskipa>\z@
-    \if@slide
-      \ltjsetxkanjiskip{0.1em}
-    \else
-      \ltjsetxkanjiskip{0.25em plus 0.15em minus 0.06em}
+  \ifltj@token@avail@scan
+    \ltjsetkanjiskip\z@ plus .1\zw minus .01\zw
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide
+        \ltjsetxkanjiskip .1em
+      \else
+       \ltjsetxkanjiskip .25em plus .15em minus .06em
+      \fi
+    \fi
+  \else
+    \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide
+        \ltjsetxkanjiskip{.1em}
+      \else
+        \ltjsetxkanjiskip{.25em plus .15em minus .06em}
+      \fi
     \fi
   \fi}
 \emergencystretch 3\zw

diff --git a/src/ltjspf.cls b/src/ltjspf.cls
index fb41e07..08985c2 100644 (file)
--- a/src/ltjspf.cls
+++ b/src/ltjspf.cls
@@ -366,13 +366,25 @@
     \fi
   \fi
   \ltj@setpar@global
-  \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
-  \@tempskipa=\ltjgetparameter{xkanjiskip}
-  \ifdim\@tempskipa>\z@
-    \if@slide
-      \ltjsetxkanjiskip{0.1em}
-    \else
-      \ltjsetxkanjiskip{0.25em plus 0.15em minus 0.06em}
+  \ifltj@token@avail@scan
+    \ltjsetkanjiskip\z@ plus .1\zw minus .01\zw
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide
+        \ltjsetxkanjiskip .1em
+      \else
+       \ltjsetxkanjiskip .25em plus .15em minus .06em
+      \fi
+    \fi
+  \else
+    \ltjsetkanjiskip{\z@ plus .1\zw minus .01\zw}
+    \@tempskipa=\ltjgetparameter{xkanjiskip}
+    \ifdim\@tempskipa>\z@
+      \if@slide
+        \ltjsetxkanjiskip{.1em}
+      \else
+        \ltjsetxkanjiskip{.25em plus .15em minus .06em}
+      \fi
     \fi
   \fi}
 \emergencystretch 3\zw

diff --git a/src/luatexja-compat.sty b/src/luatexja-compat.sty
index 6b2185e..5f519b3 100644 (file)
--- a/src/luatexja-compat.sty
+++ b/src/luatexja-compat.sty
@@ -65,23 +65,9 @@
   \expandafter\let\csname ifltj@in@latex\expandafter\endcsname
     \csname iftrue\endcsname
   \NeedsTeXFormat{LaTeX2e}
-  \ProvidesPackage{luatexja-compat}[2013/12/22 Compatibility with pTeX]
+  \ProvidesPackage{luatexja-compat}[2015/02/22 Compatibility with pTeX]
 \fi                             %</LaTeX>
 
-% ltj-compat-ptex.sty is not maintained for a long time.
-% %% If the engine is pTeX, then load the module for it and exit.
-% \edef\ltj@@tempa{\string\kansuji}
-% \edef\ltj@@tempb{\meaning\kansuji}
-% \ifx\ltj@@tempa\ltj@@tempb      %<*pTeX>
-%   \let\ltj@@compat@pTeX=t
-%   \ifltj@in@latex                 %<*LaTeX>
-%     \RequirePackage{ltj-compat-ptex}[2012/04/21]
-%   \else                           %<*!LaTeX>
-%     \input ltj-compat-ptex.sty %
-%   \fi                             %</LaTeX>
-%   \ltj@compat@AtEnd
-% \expandafter\endinput\fi\relax  %</pTeX>
-
 % Load core module if not yet.
 \ifx\luatexjacoreLoaded\@undefined
   \ifltj@in@latex                 %<*LaTeX>
@@ -113,6 +99,11 @@
 \def\sjis{\ltj@code@conv{sjis}}
 \def\ucs{\ltj@code@conv{ucs}}
 
+\ifltj@token@avail@scan %%%%%%%%%%%%%%%%%%%%%%%% new tokenlib
+\def\ltj@code@conv#1{%
+  \directlua{luatexja.compat.from_#1(luatexja.token.scan_int())}%
+}
+\else %%%%%%%%%%%%%%%%%%%%%%%% old tokenlib
 \def\ltj@code@conv#1{%
   \directlua{luatexja._cc_next = luatexja.compat.from_#1 }%
   \ltj@code@convA
@@ -124,7 +115,14 @@
 \def\ltj@code@convB#1{%
   \directlua{luatexja._cc_next(\ltj@safe@num@or{nil}{#1}) }%
 }
+\fi %%%%%%%%%%%%%%%%%%%%%%%%
+
 
+\ifltj@token@avail@scan %%%%%%%%%%%%%%%%%%%%%%%% new tokenlib
+\def\kansuji{%
+  \directlua{luatexja.compat.to_kansuji(luatexja.token.scan_int())}%
+}
+\else %%%%%%%%%%%%%%%%%%%%%%%% old tokenlib
 \def\kansuji{%
   \expandafter\expandafter\expandafter
    \ltj@kansujiA\ltj@grab@num
@@ -132,6 +130,8 @@
 \def\ltj@kansujiA#1{%
   \directlua{luatexja.compat.to_kansuji(\ltj@safe@num{#1}) }%
 }
+\fi %%%%%%%%%%%%%%%%%%%%%%%%
+
 \define@key[ltj]{japaram}{kansujichar}{%
   \expandafter\ltj@@set@stack#1:{KSJ}{0}{0x7FFFFFFF}}
 

diff --git a/src/luatexja-core.sty b/src/luatexja-core.sty
index c4c7fba..ac5f53d 100644 (file)
--- a/src/luatexja-core.sty
+++ b/src/luatexja-core.sty
@@ -65,7 +65,7 @@
   \expandafter\let\csname ifltj@in@latex\expandafter\endcsname
     \csname iftrue\endcsname
   \NeedsTeXFormat{LaTeX2e}
-  \ProvidesPackage{luatexja-core}[2014/09/24 Core of LuaTeX-ja]
+  \ProvidesPackage{luatexja-core}[2015/02/22 Core of LuaTeX-ja]
 \fi                             %</LaTeX>
 
 %% Load prerequisite packages.
@@ -73,7 +73,7 @@
 \newif\ifltj@everyhook@avail\ltj@everyhook@availfalse
 %! 外部パッケージは普通の \endlinechar で読まないと危険.
 \ifltj@in@latex                 %<*LaTeX>
-  \RequirePackage{luaotfload} [2013/05/10] % v2.2
+  \RequirePackage{luaotfload} [2013/12/31] % v2.3
   \RequirePackage{ltxcmds}    [2011/11/09] % v1.22
   \RequirePackage{pdftexcmds} [2011/11/29] % v0.20
   \RequirePackage{xkeyval}    [2012/10/14] % v2.6b
@@ -177,6 +177,16 @@
       "File `luatexja.lua' not found"))
 }
 
+
+%%% true if we can use [new]token.scan_int and co.
+\newif\ifltj@token@avail@scan
+\directlua{%
+  local bs = string.char(92)
+  if luatexja.token  then tex.sprint(bs .. 'ltj@token@avail@scantrue') end
+  print (luatexja.token)
+}
+
+
 %% \RequireLuaTeXjaSubmodule{name}[date]
 % A TeX version of luatexja.load_module(name)
 % The second optional argument [date] is simply ignored.
@@ -295,6 +305,22 @@
   \ltj@restore@globaldefs
   }
 
+\ifltj@token@avail@scan %%%%%%%%%%%%%%%%%%%%%%%% new tokenlib
+% {y,t}{ja,al}baselineshift = <dimen>
+% yjabaselineshift = <dimen>
+\define@key[ltj]{japaram}{yalbaselineshift}{
+  \directlua{tex.setattribute(luatexja.isglobal,
+    luatexbase.attributes['ltj@yablshift'], luatexja.token.scan_dimen())}#1}
+\define@key[ltj]{japaram}{yjabaselineshift}{
+  \directlua{tex.setattribute(luatexja.isglobal,
+    luatexbase.attributes['ltj@ykblshift'], luatexja.token.scan_dimen())}#1}
+\define@key[ltj]{japaram}{talbaselineshift}{
+  \directlua{tex.setattribute(luatexja.isglobal,
+    luatexbase.attributes['ltj@tablshift'], luatexja.token.scan_dimen())}#1}
+\define@key[ltj]{japaram}{tjabaselineshift}{
+  \directlua{tex.setattribute(luatexja.isglobal,
+    luatexbase.attributes['ltj@tkblshift'], luatexja.token.scan_dimen())}#1}
+\else %%%%%%%%%%%%%%%%%%%%%%%% old tokenlib
 % yalbaselineshift = <dimen>
 % yjabaselineshift = <dimen>
 \define@key[ltj]{japaram}{yalbaselineshift}{%
@@ -315,6 +341,7 @@
   \directlua{tex.setattribute(luatexja.isglobal,
     luatexbase.attributes['ltj@tkblshift'],
     \ltj@safe@dimen@or\ltj@defdimen{#1})}}
+\fi %%%%%%%%%%%%%%%%%%%%%%%%
 
 % jaxspmode = {<char_code>, <mode>}
 % mode: inhibit, preonly, postonly, allow
@@ -362,19 +389,35 @@
     \directlua{luatexja.ext_print_global()}\ltj@autoxspc=0
   \fi}
 
-% [x]]kanjiskip = <dimen>
-\define@key[ltj]{japaram}{kanjiskip}{\ltjsetkanjiskip{#1}}
-\define@key[ltj]{japaram}{xkanjiskip}{\ltjsetxkanjiskip{#1}}
-\protected\def\ltjsetkanjiskip#1{%                  % SKIP
-  \directlua{%
-    local d = \ltj@safe@glue{#1}
-    luatexja.stack.set_stack_skip(luatexja.stack_table_index.KSK, d); node.free(d)
-}\ignorespaces}
-\protected\def\ltjsetxkanjiskip#1{%                  % SKIP
-  \directlua{%
-    local d = \ltj@safe@glue{#1}
-    luatexja.stack.set_stack_skip(luatexja.stack_table_index.XSK, d); node.free(d)
-}\ignorespaces}
+% [x]]kanjiskip = <skip>
+\ifltj@token@avail@scan %%%%%%%%%%%%%%%%%%%%%%%% new tokenlib
+  \define@key[ltj]{japaram}{kanjiskip}{\message{<#1>}\ltjsetkanjiskip #1}
+  \define@key[ltj]{japaram}{xkanjiskip}{\ltjsetxkanjiskip #1}
+  \protected\def\ltjsetkanjiskip{%                  % SKIP
+    \directlua{%
+      local d = luatexja.token.scan_glue()
+      luatexja.stack.set_stack_skip(luatexja.stack_table_index.KSK, d); node.free(d)
+  }}
+  \protected\def\ltjsetxkanjiskip{%                  % SKIP
+    \directlua{%
+      local d = luatexja.token.scan_glue()
+      luatexja.stack.set_stack_skip(luatexja.stack_table_index.XSK, d); node.free(d)
+  }}
+\else %%%%%%%%%%%%%%%%%%%%%%%% old tokenlib
+  \define@key[ltj]{japaram}{kanjiskip}{\ltjsetkanjiskip{#1}}
+  \define@key[ltj]{japaram}{xkanjiskip}{\ltjsetxkanjiskip{#1}}
+  \protected\def\ltjsetkanjiskip#1{%                  % SKIP
+    \directlua{%
+      local d = \ltj@safe@glue{#1}
+      luatexja.stack.set_stack_skip(luatexja.stack_table_index.KSK, d); node.free(d)
+  }\ignorespaces}
+  \protected\def\ltjsetxkanjiskip#1{%                  % SKIP
+    \directlua{%
+      local d = \ltj@safe@glue{#1}
+      luatexja.stack.set_stack_skip(luatexja.stack_table_index.XSK, d); node.free(d)
+  }\ignorespaces}
+\else
+\fi %%%%%%%%%%%%%%%%%%%%%%%%
 
 
 % jcharwidowpenalty = <count>
@@ -673,18 +716,27 @@
 }
 
 %%%%%%%% \ltjgetwd<box_num> etc.
+\ifltj@token@avail@scan %%%%%%%%%%%%%%%%%%%%%%%% new tokenlib
+\def\ltj@@getwd{%
+  \directlua{luatexja.direction.get_box_dim('width', luatexja.token.scan_int())}%
+  \ltj@tempdima}
+\def\ltj@@getht{%
+  \directlua{luatexja.direction.get_box_dim('height', luatexja.token.scan_int()}%
+  \ltj@tempdima}
+\def\ltj@@getdp{%
+  \directlua{luatexja.direction.get_box_dim('depth', luatexja.token.scan_int())}%
+  \ltj@tempdima}
+\else %%%%%%%%%%%%%%%%%%%%%%%% old tokenlib
 \def\ltj@@getwd#1{%
   \directlua{luatexja.direction.get_box_dim('width', \the\numexpr#1\relax)}%
-  \ltj@tempdima%
-}
+  \ltj@tempdima}
 \def\ltj@@getht#1{%
   \directlua{luatexja.direction.get_box_dim('height', \the\numexpr#1\relax)}%
-  \ltj@tempdima%
-}
+  \ltj@tempdima}
 \def\ltj@@getdp#1{%
   \directlua{luatexja.direction.get_box_dim('depth', \the\numexpr#1\relax)}%
-  \ltj@tempdima%
-}
+  \ltj@tempdima}
+\fi %%%%%%%%%%%%%%%%%%%%%%%%
 \let\ltjgetwd=\ltj@@getwd\let\ltjgetht=\ltj@@getht\let\ltjgetdp=\ltj@@getdp
 
 %%%%%%%% \ltjsetwd<box_num>=<dimen> etc.

diff --git a/src/luatexja.lua b/src/luatexja.lua
index 6bd56fe..2125969 100644 (file)
--- a/src/luatexja.lua
+++ b/src/luatexja.lua
@@ -19,6 +19,16 @@ function luatexja.load_lua(fn)
    end
 end
 
+-- check token library
+if newtoken then
+   luatexja.token = newtoken
+else
+   local luatex_version = status.list().luatex_version
+   if luatex_version >80 then
+      luatexja.token = token
+   end
+end
+
 --- 以下は全ファイルで共有される定数
 local icflag_table = {}
 luatexja.icflag_table = icflag_table