From 499e88fe250201ee118902040ad27210498bef20 Mon Sep 17 00:00:00 2001 From: leo Date: Wed, 3 Jun 2009 04:57:39 +0000 Subject: [PATCH] silk assembly git-svn-id: http://www.xerial.org/svn/project/XerialJ/trunk/xerial-core@3352 ae02f08e-27ec-0310-ae8c-8ba02fe2eafd --- pom.xml | 18 ++++ src/main/java/org/xerial/silk/cui/Scan.java | 116 +++++++++++++++++++--- src/test/java/org/xerial/silk/SilkParserTest.java | 62 +----------- 3 files changed, 122 insertions(+), 74 deletions(-) diff --git a/pom.xml b/pom.xml index d712ce5..cbbfdef 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,24 @@ ${basedir}/src/main/java + + + maven-assembly-plugin + + silk + false + + jar-with-dependencies + + + + org.xerial.silk.cui.SilkMain + + true + + + + diff --git a/src/main/java/org/xerial/silk/cui/Scan.java b/src/main/java/org/xerial/silk/cui/Scan.java index 8a604dd..c7e83c4 100644 --- a/src/main/java/org/xerial/silk/cui/Scan.java +++ b/src/main/java/org/xerial/silk/cui/Scan.java @@ -24,14 +24,20 @@ //-------------------------------------- package org.xerial.silk.cui; +import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; -import org.xerial.silk.SilkStreamReader; +import org.xerial.silk.SilkEvent; +import org.xerial.silk.SilkEventHandler; +import org.xerial.silk.SilkLinePushParser; +import org.xerial.silk.SilkParser; import org.xerial.util.StopWatch; import org.xerial.util.log.Logger; import org.xerial.util.opt.Argument; +import org.xerial.util.opt.Option; import org.xerial.util.opt.Usage; -import org.xerial.util.tree.TreeEvent; +import org.xerial.util.tree.TreeEventHandlerBase; /** * Scan command @@ -44,29 +50,111 @@ public class Scan implements SilkCommand { private static Logger _logger = Logger.getLogger(Scan.class); + public static enum ScanMode { + LINE, NODE, READONLY + } + @Argument(index = 0) private String inputSilkFile = null; + @Option(symbol = "m", longName = "mode", description = "scan mode: line, node, readonly") + private ScanMode mode = ScanMode.NODE; + public void execute() throws Exception { - SilkStreamReader reader = new SilkStreamReader(new File(inputSilkFile).toURL()); - TreeEvent e; - int count = 0; - StopWatch timer = new StopWatch(); - while ((e = reader.next()) != null) + File f = new File(inputSilkFile); + final long fileSize = f.length(); + + switch (mode) + { + case NODE: + { + SilkParser parser = new SilkParser(f.toURL()); + + parser.parse(new TreeEventHandlerBase() { + + int count = 0; + StopWatch timer = new StopWatch(); + + @Override + public void init() throws Exception + { + timer.reset(); + } + + @Override + public void visitNode(String nodeName, String immediateNodeValue) throws Exception + { + count++; + if (count % 1000000 == 0) + { + double time = timer.getElapsedTime(); + double speed = count / time; + _logger.info(String.format("time=%5.2f, %,10.0f nodes/s", time, speed)); + } + + } + + @Override + public void finish() throws Exception + { + double time = timer.getElapsedTime(); + double speedPerNode = ((double) count) / time; + double speedInMBS = fileSize / 1024 / 1024 / time; + _logger.info(String + .format("time=%.2f, %,10.0f nodes/s, %3.2f MB/s", time, speedPerNode, speedInMBS)); + } + + }); + break; + } + case LINE: + { + SilkLinePushParser parser = new SilkLinePushParser(f.toURL()); + parser.parse(new SilkEventHandler() { + + int lineCount = 0; + StopWatch timer = new StopWatch(); + + public void handle(SilkEvent event) throws Exception + { + lineCount++; + if (lineCount % 100000 == 0) + { + double time = timer.getElapsedTime(); + double speed = lineCount / time; + _logger.info(String.format("time=%5.2f, line=%,10d, %,10.0f lines/s", time, lineCount, speed)); + } + + } + }); + + break; + } + case READONLY: { - count++; - if (count % 1000000 == 0) + BufferedReader reader = new BufferedReader(new FileReader(f)); + String line; + + int lineCount = 0; + StopWatch timer = new StopWatch(); + + while ((line = reader.readLine()) != null) { - long line = reader.getNumReadLine(); + lineCount++; + if (lineCount % 100000 == 0) + { + double time = timer.getElapsedTime(); + double speed = lineCount / time; + _logger.info(String.format("time=%5.2f, line=%,10d, %,10.0f lines/s", time, lineCount, speed)); + } - double time = timer.getElapsedTime(); - double speed = line / time; - _logger.info(String.format("line=%d, count=%d, time=%s, %2.2f lines/sec", line, count, time, speed)); } + break; + } } - _logger.info(String.format("elapsed time: %.1f sec.", timer.getElapsedTime())); + } public String getName() diff --git a/src/test/java/org/xerial/silk/SilkParserTest.java b/src/test/java/org/xerial/silk/SilkParserTest.java index 7bb8911..df6707a 100644 --- a/src/test/java/org/xerial/silk/SilkParserTest.java +++ b/src/test/java/org/xerial/silk/SilkParserTest.java @@ -24,15 +24,10 @@ //-------------------------------------- package org.xerial.silk; -import java.net.URL; - import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.xerial.util.FileResource; -import org.xerial.util.StopWatch; import org.xerial.util.log.Logger; -import org.xerial.util.tree.TreeEventHandlerBase; public class SilkParserTest { @@ -47,60 +42,7 @@ public class SilkParserTest {} @Test - public void testParse() throws Exception - { - final StopWatch timer = new StopWatch(); - - final SilkParser parser = new SilkParser(FileResource.open(SilkParserTest.class, "example.silk")); - parser.parse(new TreeEventHandlerBase() { - - long count = 0; - - public void visitNode(String nodeName, String immediateNodeValue) throws Exception - { - count++; - if (count % 100000 == 0) - { - double t = timer.getElapsedTime(); - long lines = parser.getNumReadLine(); - _logger.info(String.format("node count=%d, time = %.2f sec. %.0f lines/sec", count, t, lines / t)); - } - } - }); - } - - @Test - public void testPerformance() throws Exception - { - //String largeFile = "file:///c:/Users/leo/work/t2k/hdrr_hni_allaxt_revised.silk"; - String largeFile = "file:///f:/cygwin/home/leo/work/t2k/hdrr_hni_allaxt_revised.silk"; - - final SilkParser parser = new SilkParser(new URL(largeFile)); - parser.parse(new TreeEventHandlerBase() { - - long count = 0; - StopWatch timer = new StopWatch(); - - @Override - public void init() throws Exception - { - timer.reset(); - } - - public void visitNode(String nodeName, String immediateNodeValue) throws Exception - { - count++; - if (count % 100000 == 0) - { - double t = timer.getElapsedTime(); - long lines = parser.getNumReadLine(); - _logger.info(String.format("node count=%d, time = %.2f sec. %.0f lines/sec", count, t, lines / t)); - } - } - }); - - // 8700 lines/sec (Recursive Parser on Xeon 3.0GHz dual) - - } + public void dummy() + {} } -- 2.11.0