import org.xerial.silk.impl.SilkValue;
import org.xerial.silk.plugin.SilkFunctionArgument;
import org.xerial.silk.plugin.SilkFunctionPlugin;
-import org.xerial.util.ArrayDeque;
import org.xerial.util.FileResource;
import org.xerial.util.StringUtil;
import org.xerial.util.bean.TypeInfo;
import org.xerial.util.log.Logger;
import org.xerial.util.reflect.ReflectionUtil;
-import org.xerial.util.tree.TreeEvent;
import org.xerial.util.tree.TreeEventHandler;
-import org.xerial.util.tree.TreeStreamReader;
import org.xerial.util.xml.impl.TreeEventQueue;
/**
*/
public class SilkParser implements SilkEventHandler
{
- private static Logger _logger = Logger.getLogger(SilkStreamReader.class);
+ private static Logger _logger = Logger.getLogger(SilkPullParser.class);
private final SilkLinePushParser parser;
private final SilkEnv parseContext;
- private TreeEventQueue eventQueue = new TreeEventQueue();
- private final ArrayDeque<TreeStreamReader> readerStack = new ArrayDeque<TreeStreamReader>();
+ private final SilkParserConfig config;
+ private TreeEventQueue eventQueue = new TreeEventQueue();
private long numReadLine = 0;
- private final SilkParserConfig config;
-
/**
* Creates a new reader with the specified reader
*
handler.init();
parser.parse(this);
-
closeContextUpTo(parseContext.getIndentationOffset());
handler.finish();
}
+ public void parseWithoutInitAndFinish(TreeEventHandler handler) throws Exception
+ {
+ this.handler = handler;
+
+ parser.parse(this);
+ closeContextUpTo(parseContext.getIndentationOffset());
+ }
+
public void handle(SilkEvent event) throws Exception
{
try
}
- private static class FunctionReader implements TreeStreamReader
- {
- SilkFunctionPlugin plugin;
-
- public FunctionReader(SilkFunctionPlugin plugin)
- {
- this.plugin = plugin;
- }
-
- public TreeEvent peekNext() throws XerialException
- {
- return plugin.peekNext();
- }
-
- public TreeEvent next() throws XerialException
- {
- return plugin.next();
- }
- }
-
/**
* Evaluate the function
*
* @param function
- * @throws XerialException
+ * @throws Exception
*/
- private void evalFunction(SilkFunction function) throws XerialException
+ private void evalFunction(SilkFunction function) throws Exception
{
SilkFunctionPlugin plugin = getPlugin(function.getName());
if (plugin == null)
// evaluate the function
SilkEnv env = parseContext.newEnvFor(function);
plugin.init(env);
-
- readerStack.addLast(new FunctionReader(plugin));
+ plugin.eval(env, handler);
}
/**
* @author leo\r
* \r
*/\r
-public class SilkStreamReader implements TreeStreamReader\r
+public class SilkPullParser implements TreeStreamReader\r
{\r
- private static Logger _logger = Logger.getLogger(SilkStreamReader.class);\r
+ private static Logger _logger = Logger.getLogger(SilkPullParser.class);\r
\r
private final SilkParser parser;\r
private final ArrayBlockingQueue<TreeEvent> eventQueue = new ArrayBlockingQueue<TreeEvent>(10000);\r
* @param input\r
* `@throws IOException\r
*/\r
- protected SilkStreamReader(InputStream input) throws IOException\r
+ protected SilkPullParser(InputStream input) throws IOException\r
{\r
this(new InputStreamReader(input));\r
}\r
* @param input\r
* @throws IOException\r
*/\r
- protected SilkStreamReader(Reader input) throws IOException\r
+ protected SilkPullParser(Reader input) throws IOException\r
{\r
this(input, SilkEnv.newEnv());\r
}\r
* @param env\r
* @throws IOException\r
*/\r
- public SilkStreamReader(Reader input, SilkEnv env) throws IOException\r
+ public SilkPullParser(Reader input, SilkEnv env) throws IOException\r
{\r
this.parser = new SilkParser(input, env);\r
\r
* @param resourceName\r
* @throws IOException\r
*/\r
- public SilkStreamReader(String resourceBasePath, String resourceName) throws IOException\r
+ public SilkPullParser(String resourceBasePath, String resourceName) throws IOException\r
{\r
this(new BufferedReader(new InputStreamReader(SilkWalker.class.getResourceAsStream(SilkParser.getResourcePath(\r
resourceBasePath, resourceName)))), SilkEnv.newEnv(resourceBasePath));\r
* @param resourcePath\r
* @throws IOException\r
*/\r
- public SilkStreamReader(URL resourcePath) throws IOException\r
+ public SilkPullParser(URL resourcePath) throws IOException\r
{\r
this(resourcePath, SilkEnv.newEnv());\r
}\r
\r
- public SilkStreamReader(URL resource, SilkEnv env) throws IOException\r
+ public SilkPullParser(URL resource, SilkEnv env) throws IOException\r
{\r
this(new BufferedReader(new InputStreamReader(resource.openStream())), SilkEnv.newEnv(env, SilkParser\r
.getResourceBasePath(resource)));\r
*/
protected SilkWalker(InputStream input) throws IOException
{
- super(new SilkStreamReader(input));
+ super(new SilkPullParser(input));
}
/**
*/
public SilkWalker(Reader input) throws IOException
{
- super(new SilkStreamReader(input));
+ super(new SilkPullParser(input));
}
public SilkWalker(String resourceBasePath, String resourceName) throws IOException
{
- super(new SilkStreamReader(resourceBasePath, resourceName));
+ super(new SilkPullParser(resourceBasePath, resourceName));
}
public SilkWalker(URL resourcePath) throws IOException
{
- super(new SilkStreamReader(resourcePath));
+ super(new SilkPullParser(resourcePath));
}
public SilkWalker(URL resource, SilkEnv env) throws IOException
{
- super(new SilkStreamReader(resource, env));
+ super(new SilkPullParser(resource, env));
}
}
import java.io.InputStream;
import java.net.URL;
-import org.xerial.core.XerialError;
import org.xerial.core.XerialErrorCode;
import org.xerial.core.XerialException;
import org.xerial.silk.SilkEnv;
-import org.xerial.silk.SilkStreamReader;
+import org.xerial.silk.SilkParser;
import org.xerial.util.FileType;
import org.xerial.util.io.Base64OutputStream;
-import org.xerial.util.tree.TreeEvent;
-import org.xerial.util.tree.TreeStreamReader;
-import org.xerial.util.xml.impl.TreeEventQueue;
+import org.xerial.util.tree.TreeEventHandler;
/**
* <em>import</em> function
@SilkFunctionArgument
String filePath = null;
- private TreeStreamReader reader = null;
- private SilkEnv env;
-
- private static class EmptyReader implements TreeStreamReader
+ public void init(SilkEnv env) throws XerialException
{
- public TreeEvent next() throws XerialException
- {
- return null;
- }
-
- public TreeEvent peekNext() throws XerialException
- {
- return null;
- }
-
}
- public void init(SilkEnv env) throws XerialException
+ public void eval(SilkEnv env, TreeEventHandler handler) throws Exception
{
- this.env = env;
-
if (filePath == null)
{
env.getLogger().warn("no file path is specified");
- reader = new EmptyReader();
return;
}
case SILK:
case TAB:
{
- reader = new SilkStreamReader(new URL(url), env);
+ SilkParser parser = new SilkParser(new URL(url), env);
+ parser.parseWithoutInitAndFinish(handler);
break;
}
case JPEG:
case POWER_POINT:
case PNG:
{
- reader = new BinaryReader(new URL(url), env);
+ loadBinary(new URL(url), env, handler);
}
break;
default:
{
- reader = new SilkStreamReader(new URL(url), env);
+ SilkParser parser = new SilkParser(new URL(url), env);
+ parser.parseWithoutInitAndFinish(handler);
break;
}
}
+
}
catch (IOException e)
{
}
- private void validate()
+ public void loadBinary(URL path, SilkEnv env, TreeEventHandler handler) throws Exception
{
- if (env == null)
- throw new XerialError(XerialErrorCode.INVALID_STATE, "env is null");
-
- if (reader == null)
- throw new XerialError(XerialErrorCode.NOT_INITIALIZED);
-
- }
-
- public TreeEvent peekNext() throws XerialException
- {
- validate();
- return reader.peekNext();
- }
-
- public TreeEvent next() throws XerialException
- {
- validate();
- return reader.next();
- }
-
- private static class BinaryReader implements TreeStreamReader
- {
- private URL resourceURL;
- private SilkEnv env;
- private InputStream source;
- private BufferedInputStream in;
- private byte[] buffer = new byte[1024];
-
- private TreeEventQueue eventQueue = new TreeEventQueue();
- private boolean hasFinished = false;
-
- private BinaryReader(URL resourceURL, SilkEnv env)
- {
- if (resourceURL == null)
- throw new XerialError(XerialErrorCode.INVALID_INPUT);
-
- this.resourceURL = resourceURL;
- this.env = env;
-
- try
- {
- this.source = this.resourceURL.openStream();
- this.in = new BufferedInputStream(source);
-
- }
- catch (IOException e)
- {
- throw new XerialError(XerialErrorCode.IO_EXCEPTION, e);
- }
- }
-
- public TreeEvent peekNext() throws XerialException
- {
- if (eventQueue.isEmpty())
- {
- if (hasFinished)
- return null;
+ env.getLogger().debug("load binary: " + path);
- fillQueue();
- return peekNext();
- }
- else
- {
- return eventQueue.peekFirst();
- }
- }
+ InputStream source = path.openStream();
+ BufferedInputStream in = new BufferedInputStream(source);
- public TreeEvent next() throws XerialException
+ byte[] buffer = new byte[1024];
+ int readBytes = 0;
+ while ((readBytes = in.read(buffer, 0, buffer.length)) != -1)
{
- if (eventQueue.isEmpty())
+ ByteArrayOutputStream base64buffer = new ByteArrayOutputStream(readBytes);
+ Base64OutputStream base64out = new Base64OutputStream(base64buffer);
+ base64out.write(buffer, 0, readBytes);
+ base64out.flush();
+ String[] fragment = new String(base64buffer.toByteArray()).split("\\r\\n");
+ for (String each : fragment)
{
- if (hasFinished)
- return null;
-
- fillQueue();
- return next();
+ handler.text(env.getContextNode().getName(), each);
}
- else
- {
- return eventQueue.pop();
- }
- }
-
- public void fillQueue() throws XerialException
- {
- if (hasFinished)
- return;
-
- String nodeName = env.getContextNode().getName();
-
- try
- {
- int readBytes = 0;
- if ((readBytes = in.read(buffer, 0, buffer.length)) != -1)
- {
- // encode buffer data with Base64
- ByteArrayOutputStream base64buffer = new ByteArrayOutputStream(readBytes);
- Base64OutputStream base64out = new Base64OutputStream(base64buffer);
- base64out.write(buffer, 0, readBytes);
- base64out.flush();
-
- String[] fragment = new String(base64buffer.toByteArray()).split("\\r\\n");
- if (fragment == null)
- return;
- for (String each : fragment)
- {
- eventQueue.push(TreeEvent.newTextEvent(nodeName, each));
- }
- }
-
- if (readBytes == -1)
- hasFinished = true;
-
- }
- catch (IOException e)
- {
- throw new XerialException(XerialErrorCode.IO_EXCEPTION, e);
- }
-
}
}
import org.xerial.core.XerialException;
import org.xerial.silk.SilkEnv;
-import org.xerial.util.tree.TreeStreamReader;
+import org.xerial.util.tree.TreeEventHandler;
/**
* The common interface of pluggable functions that can be used in Silk format.
* @author leo
*
*/
-public interface SilkFunctionPlugin extends TreeStreamReader
+public interface SilkFunctionPlugin
{
/**
* @param env
*/
public void init(SilkEnv env) throws XerialException;
+ public void eval(SilkEnv env, TreeEventHandler handler) throws Exception;
+
}
import org.xml.sax.SAXException;\r
import org.xml.sax.XMLReader;\r
\r
-public class SilkStreamReaderTest\r
+public class SilkParserPerformanceTest\r
{\r
- private static Logger _logger = Logger.getLogger(SilkStreamReaderTest.class);\r
+ private static Logger _logger = Logger.getLogger(SilkParserPerformanceTest.class);\r
\r
- private static final URL largeFile = FileResource.find(SilkStreamReaderTest.class, "scaffold1.silk");\r
+ private static final URL largeFile = FileResource.find(SilkParserPerformanceTest.class, "scaffold1.silk");\r
private static final int largeFileSize = 30593075;\r
private static final double largeFileLines = 111965;\r
private static final int numNodes = 5826313;\r
@Test\r
public void silkStreamReaderPerformance() throws Exception\r
{\r
- SilkStreamReader reader = new SilkStreamReader(largeFile);\r
+ SilkPullParser reader = new SilkPullParser(largeFile);\r
StopWatch timer = new StopWatch();\r
int count = 0;\r
TreeEvent e;\r
OSDN Git Service
