import org.xerial.core.XerialErrorCode;
import org.xerial.core.XerialException;
import org.xerial.silk.impl.SilkLexer;
+import org.xerial.util.ArrayDeque;
import org.xerial.util.log.Logger;
/**
private final BufferedReader buffer;
private final ExecutorService threadManager;
- private final LinkedBlockingQueue<Future<List<SilkEvent>>> eventContainer;
+ private final LinkedBlockingQueue<Future<ArrayDeque<SilkEvent>>> eventContainer;
private final SilkParserConfig config;
public SilkLineFastParser(URL resourceURL) throws IOException
public SilkLineFastParser(Reader reader, SilkParserConfig config)
{
this.config = config;
- this.eventContainer = new LinkedBlockingQueue<Future<List<SilkEvent>>>(config.numWorkers);
+ this.eventContainer = new LinkedBlockingQueue<Future<ArrayDeque<SilkEvent>>>(config.numWorkers);
if (reader.getClass().isAssignableFrom(BufferedReader.class))
buffer = BufferedReader.class.cast(reader);
int lineCount = 0;
while (lineCount < config.numLinesInBlock)
{
- lineCount++;
+
String line = buffer.readLine();
if (line == null)
{
foundEOF = true;
break;
}
+ lineCount++;
cache.add(line);
}
// map the input
Mapper map = new Mapper(workerCount++, cache);
- Future<List<SilkEvent>> future = threadManager.submit(map);
+ Future<ArrayDeque<SilkEvent>> future = threadManager.submit(map);
eventContainer.put(future);
}
}
{
Thread.sleep(1);
}
+
reducerTask.cancel(true);
}
}
- private class Mapper implements Callable<List<SilkEvent>>
+ private class Mapper implements Callable<ArrayDeque<SilkEvent>>
{
final List<String> cache;
- final List<SilkEvent> eventQueue;
+ final ArrayDeque<SilkEvent> eventQueue;
final SilkLexer lexer = new SilkLexer();
final int lsn;
{
this.lsn = lsn;
this.cache = cache;
- eventQueue = new ArrayList<SilkEvent>(cache.size());
+ eventQueue = new ArrayDeque<SilkEvent>(cache.size());
}
- public List<SilkEvent> call() throws Exception
+ public ArrayDeque<SilkEvent> call() throws Exception
{
for (int cursor = 0; cursor < cache.size(); cursor++)
{
private class Reducer implements Callable<Void>
{
private SilkEventHandler handler;
+ private ArrayDeque<SilkEvent> eventQueue;
public Reducer(SilkEventHandler handler)
{
{
while (true)
{
- Future<List<SilkEvent>> container = eventContainer.take();
- List<SilkEvent> eventQueue = container.get();
- for (SilkEvent e : eventQueue)
+ try
+ {
+ Future<ArrayDeque<SilkEvent>> container = eventContainer.take();
+ eventQueue = container.get();
+ }
+ catch (InterruptedException e)
{
- handler.handle(e);
+ _logger.info("interrupted");
+ return null;
+ }
+
+ while (!eventQueue.isEmpty())
+ {
+ SilkEvent e = null;
+ try
+ {
+ e = eventQueue.getFirst();
+ }
+ finally
+ {
+ if (e != null)
+ handler.handle(e);
+ eventQueue.removeFirst();
+ }
}
}
}
- catch (InterruptedException e)
+ finally
{
+ // handle the remaining events
+ for (SilkEvent e : eventQueue)
+ handler.handle(e);
+
handler.handle(new SilkEvent(SilkEventType.END_OF_FILE, null));
}
-
- return null;
}
}
//--------------------------------------
package org.xerial.json;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import org.junit.After;
import org.junit.Before;
assertEquals("leo", p.getString("name"));
}
- @Test
- public void testParse() throws JSONException
+ final int N = 5000;
+
+ public String createSampleJSONArrayData()
{
// generate a sample JSON array
StringBuilder sample = new StringBuilder();
sample.append("[");
int i = 0;
- final int N = 5000;
+
for (; i < N - 1; i++)
{
sample.append(i);
sample.append(i);
sample.append("]");
- String json = sample.toString();
+ return sample.toString();
+ }
+ @Test
+ public void testParse() throws JSONException
+ {
+ String json = createSampleJSONArrayData();
StopWatch timer = new StopWatch();
for (int n = 0; n < 500; n++)
{
}
_logger.info("time: " + timer.getElapsedTime());
- // i:1000, n:100 time=18.4 sec (2009.4.23 using ANTLR JSON.g)
- // i:1000, n:100 time=2.248 (2009. 4.23 using JSONTokener)
-
}
@Test
public void testParseANTLRLexer() throws JSONException
{
- // generate a sample JSON array
- StringBuilder sample = new StringBuilder();
- sample.append("[");
- int i = 0;
- final int N = 5000;
- for (; i < N - 1; i++)
- {
- sample.append(i);
- sample.append(",");
- }
- sample.append(i);
- sample.append("]");
-
- String json = sample.toString();
+ String json = createSampleJSONArrayData();
StopWatch timer = new StopWatch();
for (int n = 0; n < 500; n++)
}
_logger.info("time: " + timer.getElapsedTime());
- // i:1000, n:100 time=18.4 sec (2009.4.23 using ANTLR JSON.g)
- // i:1000, n:100 time=2.248 (2009. 4.23 using JSONTokener)
-
}
}
public void setUp() throws Exception\r
{\r
config = new SilkParserConfig();\r
- config.bufferSize = 1024 * 1024 * 16; // 16MB\r
+ config.bufferSize = 1024 * 1024 * 8; // 8MB\r
+ config.numWorkers = 1;\r
}\r
\r
@After\r
public void tearDown() throws Exception\r
{}\r
\r
- //private static final String largeFile = "file:///c:/Users/leo/work/t2k/hdrr_hni_allaxt_revised.silk";\r
-\r
- //private static final String largeFile = "file:///f:/cygwin/home/leo/work/t2k/hdrr_hni_allaxt_revised.silk";\r
-\r
- //private static final String largeFile = "file:///d:/tmp/hdrr_hni_allaxt_revised.silk";\r
- //private static final String largeFile = "file:///f:/cygwin/home/leo/work/t2k/hdrr_hni_allaxt_revised.silk";\r
-\r
- @Test\r
- public void dummy()\r
- {\r
-\r
- }\r
-\r
@Ignore\r
@Test\r
public void xmlReadTest() throws Exception\r
\r
}\r
\r
- int count = 0;\r
-\r
- @Test\r
- public void parserPerformance() throws Exception\r
- {\r
- SilkParser parser = new SilkParser(largeFile, config);\r
- final StopWatch timer = new StopWatch();\r
-\r
- count = 0;\r
- parser.parse(new TreeEventHandlerBase() {\r
-\r
- @Override\r
- public void visitNode(String nodeName, String immediateNodeValue) throws Exception\r
- {\r
- count++;\r
- if (count % 1000000 == 0)\r
- reportNodeCountStat(count, timer.getElapsedTime());\r
- }\r
-\r
- });\r
-\r
- reportTotalSpeed("SilkParser", count, timer.getElapsedTime());\r
-\r
- }\r
-\r
@Ignore\r
@Test\r
public void pullParserPerformanceTest() throws Exception\r
reportTotalSpeed("SilkFastPushParser", timer.getElapsedTime());\r
}\r
\r
+ int count = 0;\r
+\r
+ @Test\r
+ public void parserPerformance() throws Exception\r
+ {\r
+ SilkParser parser = new SilkParser(largeFile, config);\r
+ final StopWatch timer = new StopWatch();\r
+\r
+ count = 0;\r
+ parser.parse(new TreeEventHandlerBase() {\r
+\r
+ @Override\r
+ public void visitNode(String nodeName, String immediateNodeValue) throws Exception\r
+ {\r
+ count++;\r
+ if (count % 1000000 == 0)\r
+ reportNodeCountStat(count, timer.getElapsedTime());\r
+ }\r
+\r
+ });\r
+\r
+ reportTotalSpeed("SilkParser", count, timer.getElapsedTime());\r
+\r
+ }\r
+\r
}\r
OSDN Git Service
