From a1368af7e8c52fc70ef78e2749d6c72703ea876b Mon Sep 17 00:00:00 2001 From: leo Date: Fri, 5 Jun 2009 06:10:40 +0000 Subject: [PATCH] changed silk function from pull-style to push style git-svn-id: http://www.xerial.org/svn/project/XerialJ/trunk/xerial-core@3362 ae02f08e-27ec-0310-ae8c-8ba02fe2eafd --- src/main/java/org/xerial/silk/SilkParser.java | 47 ++---- .../{SilkStreamReader.java => SilkPullParser.java} | 16 +- src/main/java/org/xerial/silk/SilkWalker.java | 10 +- src/main/java/org/xerial/silk/plugin/Import.java | 165 +++------------------ .../org/xerial/silk/plugin/SilkFunctionPlugin.java | 6 +- ...derTest.java => SilkParserPerformanceTest.java} | 8 +- 6 files changed, 59 insertions(+), 193 deletions(-) rename src/main/java/org/xerial/silk/{SilkStreamReader.java => SilkPullParser.java} (88%) rename src/test/java/org/xerial/silk/{SilkStreamReaderTest.java => SilkParserPerformanceTest.java} (94%) diff --git a/src/main/java/org/xerial/silk/SilkParser.java b/src/main/java/org/xerial/silk/SilkParser.java index 9c5bee2..12bd941 100644 --- a/src/main/java/org/xerial/silk/SilkParser.java +++ b/src/main/java/org/xerial/silk/SilkParser.java @@ -57,15 +57,12 @@ import org.xerial.silk.impl.SilkNodeOccurrence; import org.xerial.silk.impl.SilkValue; import org.xerial.silk.plugin.SilkFunctionArgument; import org.xerial.silk.plugin.SilkFunctionPlugin; -import org.xerial.util.ArrayDeque; import org.xerial.util.FileResource; import org.xerial.util.StringUtil; import org.xerial.util.bean.TypeInfo; import org.xerial.util.log.Logger; import org.xerial.util.reflect.ReflectionUtil; -import org.xerial.util.tree.TreeEvent; import org.xerial.util.tree.TreeEventHandler; -import org.xerial.util.tree.TreeStreamReader; import org.xerial.util.xml.impl.TreeEventQueue; /** @@ -76,17 +73,15 @@ import org.xerial.util.xml.impl.TreeEventQueue; */ public class SilkParser implements SilkEventHandler { - private static Logger _logger = Logger.getLogger(SilkStreamReader.class); + private static Logger _logger = Logger.getLogger(SilkPullParser.class); private final SilkLinePushParser parser; private final SilkEnv parseContext; - private TreeEventQueue eventQueue = new TreeEventQueue(); - private final ArrayDeque readerStack = new ArrayDeque(); + private final SilkParserConfig config; + private TreeEventQueue eventQueue = new TreeEventQueue(); private long numReadLine = 0; - private final SilkParserConfig config; - /** * Creates a new reader with the specified reader * @@ -175,12 +170,19 @@ public class SilkParser implements SilkEventHandler handler.init(); parser.parse(this); - closeContextUpTo(parseContext.getIndentationOffset()); handler.finish(); } + public void parseWithoutInitAndFinish(TreeEventHandler handler) throws Exception + { + this.handler = handler; + + parser.parse(this); + closeContextUpTo(parseContext.getIndentationOffset()); + } + public void handle(SilkEvent event) throws Exception { try @@ -539,33 +541,13 @@ public class SilkParser implements SilkEventHandler } - private static class FunctionReader implements TreeStreamReader - { - SilkFunctionPlugin plugin; - - public FunctionReader(SilkFunctionPlugin plugin) - { - this.plugin = plugin; - } - - public TreeEvent peekNext() throws XerialException - { - return plugin.peekNext(); - } - - public TreeEvent next() throws XerialException - { - return plugin.next(); - } - } - /** * Evaluate the function * * @param function - * @throws XerialException + * @throws Exception */ - private void evalFunction(SilkFunction function) throws XerialException + private void evalFunction(SilkFunction function) throws Exception { SilkFunctionPlugin plugin = getPlugin(function.getName()); if (plugin == null) @@ -579,8 +561,7 @@ public class SilkParser implements SilkEventHandler // evaluate the function SilkEnv env = parseContext.newEnvFor(function); plugin.init(env); - - readerStack.addLast(new FunctionReader(plugin)); + plugin.eval(env, handler); } /** diff --git a/src/main/java/org/xerial/silk/SilkStreamReader.java b/src/main/java/org/xerial/silk/SilkPullParser.java similarity index 88% rename from src/main/java/org/xerial/silk/SilkStreamReader.java rename to src/main/java/org/xerial/silk/SilkPullParser.java index e9a02b7..0f02443 100644 --- a/src/main/java/org/xerial/silk/SilkStreamReader.java +++ b/src/main/java/org/xerial/silk/SilkPullParser.java @@ -49,9 +49,9 @@ import org.xerial.util.tree.TreeStreamReader; * @author leo * */ -public class SilkStreamReader implements TreeStreamReader +public class SilkPullParser implements TreeStreamReader { - private static Logger _logger = Logger.getLogger(SilkStreamReader.class); + private static Logger _logger = Logger.getLogger(SilkPullParser.class); private final SilkParser parser; private final ArrayBlockingQueue eventQueue = new ArrayBlockingQueue(10000); @@ -68,7 +68,7 @@ public class SilkStreamReader implements TreeStreamReader * @param input * `@throws IOException */ - protected SilkStreamReader(InputStream input) throws IOException + protected SilkPullParser(InputStream input) throws IOException { this(new InputStreamReader(input)); } @@ -79,7 +79,7 @@ public class SilkStreamReader implements TreeStreamReader * @param input * @throws IOException */ - protected SilkStreamReader(Reader input) throws IOException + protected SilkPullParser(Reader input) throws IOException { this(input, SilkEnv.newEnv()); } @@ -91,7 +91,7 @@ public class SilkStreamReader implements TreeStreamReader * @param env * @throws IOException */ - public SilkStreamReader(Reader input, SilkEnv env) throws IOException + public SilkPullParser(Reader input, SilkEnv env) throws IOException { this.parser = new SilkParser(input, env); @@ -152,7 +152,7 @@ public class SilkStreamReader implements TreeStreamReader * @param resourceName * @throws IOException */ - public SilkStreamReader(String resourceBasePath, String resourceName) throws IOException + public SilkPullParser(String resourceBasePath, String resourceName) throws IOException { this(new BufferedReader(new InputStreamReader(SilkWalker.class.getResourceAsStream(SilkParser.getResourcePath( resourceBasePath, resourceName)))), SilkEnv.newEnv(resourceBasePath)); @@ -164,12 +164,12 @@ public class SilkStreamReader implements TreeStreamReader * @param resourcePath * @throws IOException */ - public SilkStreamReader(URL resourcePath) throws IOException + public SilkPullParser(URL resourcePath) throws IOException { this(resourcePath, SilkEnv.newEnv()); } - public SilkStreamReader(URL resource, SilkEnv env) throws IOException + public SilkPullParser(URL resource, SilkEnv env) throws IOException { this(new BufferedReader(new InputStreamReader(resource.openStream())), SilkEnv.newEnv(env, SilkParser .getResourceBasePath(resource))); diff --git a/src/main/java/org/xerial/silk/SilkWalker.java b/src/main/java/org/xerial/silk/SilkWalker.java index e8cd765..460cd86 100644 --- a/src/main/java/org/xerial/silk/SilkWalker.java +++ b/src/main/java/org/xerial/silk/SilkWalker.java @@ -49,7 +49,7 @@ public class SilkWalker extends TreeWalkerImpl */ protected SilkWalker(InputStream input) throws IOException { - super(new SilkStreamReader(input)); + super(new SilkPullParser(input)); } /** @@ -60,22 +60,22 @@ public class SilkWalker extends TreeWalkerImpl */ public SilkWalker(Reader input) throws IOException { - super(new SilkStreamReader(input)); + super(new SilkPullParser(input)); } public SilkWalker(String resourceBasePath, String resourceName) throws IOException { - super(new SilkStreamReader(resourceBasePath, resourceName)); + super(new SilkPullParser(resourceBasePath, resourceName)); } public SilkWalker(URL resourcePath) throws IOException { - super(new SilkStreamReader(resourcePath)); + super(new SilkPullParser(resourcePath)); } public SilkWalker(URL resource, SilkEnv env) throws IOException { - super(new SilkStreamReader(resource, env)); + super(new SilkPullParser(resource, env)); } } diff --git a/src/main/java/org/xerial/silk/plugin/Import.java b/src/main/java/org/xerial/silk/plugin/Import.java index d935540..1c55a54 100644 --- a/src/main/java/org/xerial/silk/plugin/Import.java +++ b/src/main/java/org/xerial/silk/plugin/Import.java @@ -30,16 +30,13 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; -import org.xerial.core.XerialError; import org.xerial.core.XerialErrorCode; import org.xerial.core.XerialException; import org.xerial.silk.SilkEnv; -import org.xerial.silk.SilkStreamReader; +import org.xerial.silk.SilkParser; import org.xerial.util.FileType; import org.xerial.util.io.Base64OutputStream; -import org.xerial.util.tree.TreeEvent; -import org.xerial.util.tree.TreeStreamReader; -import org.xerial.util.xml.impl.TreeEventQueue; +import org.xerial.util.tree.TreeEventHandler; /** * import function @@ -53,32 +50,16 @@ public class Import implements SilkFunctionPlugin @SilkFunctionArgument String filePath = null; - private TreeStreamReader reader = null; - private SilkEnv env; - - private static class EmptyReader implements TreeStreamReader + public void init(SilkEnv env) throws XerialException { - public TreeEvent next() throws XerialException - { - return null; - } - - public TreeEvent peekNext() throws XerialException - { - return null; - } - } - public void init(SilkEnv env) throws XerialException + public void eval(SilkEnv env, TreeEventHandler handler) throws Exception { - this.env = env; - if (filePath == null) { env.getLogger().warn("no file path is specified"); - reader = new EmptyReader(); return; } @@ -95,7 +76,8 @@ public class Import implements SilkFunctionPlugin case SILK: case TAB: { - reader = new SilkStreamReader(new URL(url), env); + SilkParser parser = new SilkParser(new URL(url), env); + parser.parseWithoutInitAndFinish(handler); break; } case JPEG: @@ -109,15 +91,17 @@ public class Import implements SilkFunctionPlugin case POWER_POINT: case PNG: { - reader = new BinaryReader(new URL(url), env); + loadBinary(new URL(url), env, handler); } break; default: { - reader = new SilkStreamReader(new URL(url), env); + SilkParser parser = new SilkParser(new URL(url), env); + parser.parseWithoutInitAndFinish(handler); break; } } + } catch (IOException e) { @@ -126,127 +110,26 @@ public class Import implements SilkFunctionPlugin } - private void validate() + public void loadBinary(URL path, SilkEnv env, TreeEventHandler handler) throws Exception { - if (env == null) - throw new XerialError(XerialErrorCode.INVALID_STATE, "env is null"); - - if (reader == null) - throw new XerialError(XerialErrorCode.NOT_INITIALIZED); - - } - - public TreeEvent peekNext() throws XerialException - { - validate(); - return reader.peekNext(); - } - - public TreeEvent next() throws XerialException - { - validate(); - return reader.next(); - } - - private static class BinaryReader implements TreeStreamReader - { - private URL resourceURL; - private SilkEnv env; - private InputStream source; - private BufferedInputStream in; - private byte[] buffer = new byte[1024]; - - private TreeEventQueue eventQueue = new TreeEventQueue(); - private boolean hasFinished = false; - - private BinaryReader(URL resourceURL, SilkEnv env) - { - if (resourceURL == null) - throw new XerialError(XerialErrorCode.INVALID_INPUT); - - this.resourceURL = resourceURL; - this.env = env; - - try - { - this.source = this.resourceURL.openStream(); - this.in = new BufferedInputStream(source); - - } - catch (IOException e) - { - throw new XerialError(XerialErrorCode.IO_EXCEPTION, e); - } - } - - public TreeEvent peekNext() throws XerialException - { - if (eventQueue.isEmpty()) - { - if (hasFinished) - return null; + env.getLogger().debug("load binary: " + path); - fillQueue(); - return peekNext(); - } - else - { - return eventQueue.peekFirst(); - } - } + InputStream source = path.openStream(); + BufferedInputStream in = new BufferedInputStream(source); - public TreeEvent next() throws XerialException + byte[] buffer = new byte[1024]; + int readBytes = 0; + while ((readBytes = in.read(buffer, 0, buffer.length)) != -1) { - if (eventQueue.isEmpty()) + ByteArrayOutputStream base64buffer = new ByteArrayOutputStream(readBytes); + Base64OutputStream base64out = new Base64OutputStream(base64buffer); + base64out.write(buffer, 0, readBytes); + base64out.flush(); + String[] fragment = new String(base64buffer.toByteArray()).split("\\r\\n"); + for (String each : fragment) { - if (hasFinished) - return null; - - fillQueue(); - return next(); + handler.text(env.getContextNode().getName(), each); } - else - { - return eventQueue.pop(); - } - } - - public void fillQueue() throws XerialException - { - if (hasFinished) - return; - - String nodeName = env.getContextNode().getName(); - - try - { - int readBytes = 0; - if ((readBytes = in.read(buffer, 0, buffer.length)) != -1) - { - // encode buffer data with Base64 - ByteArrayOutputStream base64buffer = new ByteArrayOutputStream(readBytes); - Base64OutputStream base64out = new Base64OutputStream(base64buffer); - base64out.write(buffer, 0, readBytes); - base64out.flush(); - - String[] fragment = new String(base64buffer.toByteArray()).split("\\r\\n"); - if (fragment == null) - return; - for (String each : fragment) - { - eventQueue.push(TreeEvent.newTextEvent(nodeName, each)); - } - } - - if (readBytes == -1) - hasFinished = true; - - } - catch (IOException e) - { - throw new XerialException(XerialErrorCode.IO_EXCEPTION, e); - } - } } diff --git a/src/main/java/org/xerial/silk/plugin/SilkFunctionPlugin.java b/src/main/java/org/xerial/silk/plugin/SilkFunctionPlugin.java index 68d22e5..f34c995 100644 --- a/src/main/java/org/xerial/silk/plugin/SilkFunctionPlugin.java +++ b/src/main/java/org/xerial/silk/plugin/SilkFunctionPlugin.java @@ -26,7 +26,7 @@ package org.xerial.silk.plugin; import org.xerial.core.XerialException; import org.xerial.silk.SilkEnv; -import org.xerial.util.tree.TreeStreamReader; +import org.xerial.util.tree.TreeEventHandler; /** * The common interface of pluggable functions that can be used in Silk format. @@ -34,7 +34,7 @@ import org.xerial.util.tree.TreeStreamReader; * @author leo * */ -public interface SilkFunctionPlugin extends TreeStreamReader +public interface SilkFunctionPlugin { /** * @param env @@ -43,4 +43,6 @@ public interface SilkFunctionPlugin extends TreeStreamReader */ public void init(SilkEnv env) throws XerialException; + public void eval(SilkEnv env, TreeEventHandler handler) throws Exception; + } diff --git a/src/test/java/org/xerial/silk/SilkStreamReaderTest.java b/src/test/java/org/xerial/silk/SilkParserPerformanceTest.java similarity index 94% rename from src/test/java/org/xerial/silk/SilkStreamReaderTest.java rename to src/test/java/org/xerial/silk/SilkParserPerformanceTest.java index 8da2196..0267cc8 100644 --- a/src/test/java/org/xerial/silk/SilkStreamReaderTest.java +++ b/src/test/java/org/xerial/silk/SilkParserPerformanceTest.java @@ -51,11 +51,11 @@ import org.xml.sax.Locator; import org.xml.sax.SAXException; import org.xml.sax.XMLReader; -public class SilkStreamReaderTest +public class SilkParserPerformanceTest { - private static Logger _logger = Logger.getLogger(SilkStreamReaderTest.class); + private static Logger _logger = Logger.getLogger(SilkParserPerformanceTest.class); - private static final URL largeFile = FileResource.find(SilkStreamReaderTest.class, "scaffold1.silk"); + private static final URL largeFile = FileResource.find(SilkParserPerformanceTest.class, "scaffold1.silk"); private static final int largeFileSize = 30593075; private static final double largeFileLines = 111965; private static final int numNodes = 5826313; @@ -264,7 +264,7 @@ public class SilkStreamReaderTest @Test public void silkStreamReaderPerformance() throws Exception { - SilkStreamReader reader = new SilkStreamReader(largeFile); + SilkPullParser reader = new SilkPullParser(largeFile); StopWatch timer = new StopWatch(); int count = 0; TreeEvent e; -- 2.11.0