{
public struct CharInfo
{
- #region Const/Field/Property
+ #region Const/Field/Property
- private readonly uint bits;
+ private uint bits;
/// <summary>
/// 互換カテゴリ
{
string fileName = Path.Combine(dicDir, CharPropertyFile);
- using (FileStream stream = new FileStream(fileName, FileMode.Open, FileAccess.Read))
- using (BinaryReader reader = new BinaryReader(stream))
+ using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+ using (var reader = new BinaryReader(stream))
{
this.Open(reader, fileName);
}
\r
#if MMF_MTX\r
private MemoryMappedFile mmf;\r
- private MemoryMappedViewAccessor matrix;\r
+ private MemoryMappedViewAccessor mmva;\r
+ private unsafe short* matrix;\r
#else\r
private short[] matrix;\r
#endif\r
}\r
\r
#if MMF_MTX\r
-\r
- public void Open(string fileName)\r
+ public unsafe void Open(string fileName)\r
{\r
- //MMFインスタンスを生成するが、後でDisposeするために保持しておく\r
- this.mmf = MemoryMappedFile.CreateFromFile(fileName, FileMode.Open,\r
- null, 0L, MemoryMappedFileAccess.Read);\r
- this.Open(this.mmf);\r
- }\r
+ this.mmf = MemoryMappedFile.CreateFromFile(fileName, FileMode.Open, null, 0L, MemoryMappedFileAccess.Read);\r
+ this.mmva = this.mmf.CreateViewAccessor(0L, 0L, MemoryMappedFileAccess.Read);\r
\r
- public void Open(MemoryMappedFile mmf)\r
- {\r
- using (MemoryMappedViewStream stream = mmf.CreateViewStream(\r
- 0L, 0L, MemoryMappedFileAccess.Read))\r
- using (BinaryReader reader = new BinaryReader(stream))\r
+ byte* ptr = null;\r
+ this.mmva.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);\r
+\r
+ using (var stream = mmf.CreateViewStream(0L, 0L, MemoryMappedFileAccess.Read))\r
+ using (var reader = new BinaryReader(stream))\r
{\r
this.LSize = reader.ReadUInt16();\r
this.RSize = reader.ReadUInt16();\r
\r
- long offset = stream.Position;\r
- long size = this.LSize * this.RSize * sizeof(short);\r
- this.matrix = mmf.CreateViewAccessor(offset, size, MemoryMappedFileAccess.Read);\r
+ long fSize = stream.Position + sizeof(short) * this.LSize * this.RSize;\r
+ if (this.mmva.Capacity < fSize)\r
+ throw new MeCabInvalidFileException("file size is invalid", fileName);\r
+\r
+ ptr += stream.Position;\r
+ this.matrix = (short*)ptr;\r
}\r
}\r
-\r
#else\r
-\r
public void Open(string fileName)\r
{\r
- using (FileStream stream = new FileStream(fileName, FileMode.Open, FileAccess.Read))\r
- using (BinaryReader reader = new BinaryReader(stream))\r
+ using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read))\r
+ using (var reader = new BinaryReader(stream))\r
{\r
this.Open(reader, fileName);\r
}\r
if (reader.BaseStream.ReadByte() != -1)\r
throw new MeCabInvalidFileException("file size is invalid", fileName);\r
}\r
-\r
#endif\r
\r
#endregion\r
\r
#region Cost\r
\r
- public int Cost(MeCabNode lNode, MeCabNode rNode)\r
+ public unsafe int Cost(MeCabNode lNode, MeCabNode rNode)\r
{\r
int pos = lNode.RCAttr + this.LSize * rNode.LCAttr;\r
-\r
-#if MMF_MTX\r
- return this.matrix.ReadInt16(pos * sizeof(short)) + rNode.WCost;\r
-#else\r
return this.matrix[pos] + rNode.WCost;\r
-#endif\r
}\r
\r
#endregion\r
if (disposing)\r
{\r
#if MMF_MTX\r
- if (this.mmf != null) this.mmf.Dispose();\r
- if (this.matrix != null) this.matrix.Dispose();\r
+ if (this.mmva != null)\r
+ {\r
+ this.mmva.SafeMemoryMappedViewHandle.ReleasePointer();\r
+ this.mmva.Dispose();\r
+ }\r
+\r
+ if (this.mmf != null)\r
+ this.mmf.Dispose();\r
#endif\r
}\r
\r
using System;\r
using System.Collections.Generic;\r
using System.Text;\r
+#if !MMF_DIC\r
using System.IO;\r
-#if MMF_DIC\r
-using System.IO.MemoryMappedFiles;\r
#endif\r
\r
namespace NMeCab.Core\r
/// <summary>\r
/// Double-Array Trie の実装\r
/// </summary>\r
- public class DoubleArray : IDisposable\r
+ public class DoubleArray\r
{\r
#region Array\r
\r
private struct Unit\r
{\r
- public readonly int Base;\r
- public readonly uint Check;\r
-\r
- public Unit(int b, uint c)\r
- {\r
- this.Base = b;\r
- this.Check = c;\r
- }\r
+#pragma warning disable 0649\r
+ public int Base;\r
+ public uint Check;\r
+#pragma warning restore 0649\r
}\r
\r
public const int UnitSize = sizeof(int) + sizeof(uint);\r
\r
#if MMF_DIC\r
\r
- private MemoryMappedViewAccessor accessor;\r
+ private unsafe Unit* array;\r
\r
public int Size\r
{\r
- get { return (int)(this.accessor.Capacity) / UnitSize; }\r
+ get { return this.TotalSize / UnitSize; }\r
}\r
\r
- public int TotalSize\r
- {\r
- get { return (int)(this.accessor.Capacity); }\r
- }\r
+ public int TotalSize { get; private set; }\r
\r
#else\r
\r
\r
#if MMF_DIC\r
\r
- public void Open(MemoryMappedFile mmf, long offset, long size)\r
+ public unsafe void Open(byte* ptr, int size)\r
{\r
- this.accessor = mmf.CreateViewAccessor(offset, size, MemoryMappedFileAccess.Read);\r
+ this.array = (Unit*)ptr;\r
+ this.TotalSize = size;\r
}\r
\r
#else\r
\r
- public void Open(BinaryReader reader, uint size)\r
+ public void Open(BinaryReader reader, int size)\r
{\r
this.array = new Unit[size / UnitSize];\r
\r
for (int i = 0; i < array.Length; i++)\r
{\r
- this.array[i] = new Unit(reader.ReadInt32(), reader.ReadUInt32());\r
+ this.array[i] = new Unit()\r
+ {\r
+ Base = reader.ReadInt32(),\r
+ Check = reader.ReadUInt32()\r
+ };\r
}\r
}\r
\r
public struct ResultPair\r
{\r
public int Value;\r
-\r
public int Length;\r
-\r
- public ResultPair(int r, int t)\r
- {\r
- this.Value = r;\r
- this.Length = t;\r
- }\r
- }\r
-\r
- public unsafe void ExactMatchSearch(byte* key, ResultPair* result, int len, int nodePos)\r
- {\r
- *result = this.ExactMatchSearch(key, len, nodePos);\r
}\r
\r
public unsafe ResultPair ExactMatchSearch(byte* key, int len, int nodePos)\r
{\r
- int b = this.ReadBase(nodePos);\r
- Unit p;\r
+ int b = this.array[nodePos].Base;\r
+ int p;\r
\r
for (int i = 0; i < len; i++)\r
{\r
- this.ReadUnit(b + key[i] + 1, out p);\r
- if (b == p.Check)\r
+ p = b + key[i] + 1;\r
+ if (b == this.array[p].Check)\r
{\r
- b = p.Base;\r
+ b = this.array[p].Base;\r
}\r
else\r
{\r
- return new ResultPair(-1, 0);\r
+ return new ResultPair() { Value = -1, Length = 0 };\r
}\r
}\r
\r
- this.ReadUnit(b, out p);\r
- int n = p.Base;\r
- if (b == p.Check && n < 0)\r
+ p = b;\r
+ int n = this.array[b].Base;\r
+ if (b == this.array[p].Check && n < 0)\r
{\r
- return new ResultPair(-n - 1, len);\r
+ return new ResultPair() { Value = -n - 1, Length = 0 };\r
}\r
\r
- return new ResultPair(-1, 0);\r
+ return new ResultPair() { Value = -1, Length = 0 };\r
}\r
\r
public unsafe int CommonPrefixSearch(byte* key, ResultPair* result, int resultLen, int len, int nodePos = 0)\r
{\r
- int b = this.ReadBase(nodePos);\r
+ int b = this.array[nodePos].Base;\r
int num = 0;\r
int n;\r
- Unit p;\r
+ int p;\r
\r
for (int i = 0; i < len; i++)\r
{\r
- this.ReadUnit(b, out p);\r
- n = p.Base;\r
+ p = b;\r
+ n = this.array[p].Base;\r
\r
- if (b == p.Check && n < 0)\r
+ if (b == this.array[p].Check && n < 0)\r
{\r
- if (num < resultLen) result[num] = new ResultPair(-n - 1, i);\r
+ if (num < resultLen)\r
+ result[num] = new ResultPair() { Value = -n - 1, Length = i };\r
num++;\r
}\r
\r
- this.ReadUnit(b + key[i] + 1, out p);\r
- if (b == p.Check)\r
+ p = b + key[i] + 1;\r
+ if (b == this.array[p].Check)\r
{\r
- b = p.Base;\r
+ b = this.array[p].Base;\r
}\r
else\r
{\r
}\r
}\r
\r
- this.ReadUnit(b, out p);\r
- n = p.Base;\r
+ p = b;\r
+ n = this.array[p].Base;\r
\r
- if (b == p.Check && n < 0)\r
+ if (b == this.array[p].Check && n < 0)\r
{\r
- if (num < resultLen) result[num] = new ResultPair(-n - 1, len);\r
+ if (num < resultLen)\r
+ result[num] = new ResultPair() { Value = -n - 1, Length = len };\r
num++;\r
}\r
\r
return num;\r
}\r
\r
-\r
-\r
- private int ReadBase(int pos)\r
- {\r
-#if MMF_DIC\r
- return this.accessor.ReadInt32(pos * UnitSize);\r
-#else\r
- return this.array[pos].Base;\r
-#endif\r
- }\r
-\r
- private void ReadUnit(int pos, out Unit unit)\r
- {\r
-#if MMF_DIC\r
- this.accessor.Read<Unit>(pos * UnitSize, out unit);\r
-#else\r
- unit = this.array[pos];\r
-#endif\r
- }\r
-\r
- #endregion\r
-\r
- #region Dispose\r
-\r
- private bool disposed;\r
-\r
- public void Dispose()\r
- {\r
- this.Dispose(true);\r
- GC.SuppressFinalize(this);\r
- }\r
-\r
- protected virtual void Dispose(bool disposing)\r
- {\r
- if (disposed) return;\r
-\r
- if (disposing)\r
- {\r
-#if MMF_DIC\r
- if (this.accessor != null) this.accessor.Dispose();\r
-#endif\r
- }\r
-\r
- this.disposed = true;\r
- }\r
-\r
- ~DoubleArray()\r
- {\r
- this.Dispose(false);\r
- }\r
-\r
#endregion\r
}\r
}\r
#if MMF_DIC
private MemoryMappedFile mmf;
- private MemoryMappedViewAccessor tokens;
- private MemoryMappedViewAccessor features;
+ private MemoryMappedViewAccessor mmva;
+ private unsafe Token* tokens;
+ private unsafe byte* features;
#else
private Token[] tokens;
private byte[] features;
#endif
- private DoubleArray da = new DoubleArray();
+ private readonly DoubleArray da = new DoubleArray();
private Encoding encoding;
#if MMF_DIC
- public void Open(string filePath)
+ public unsafe void Open(string fileName)
{
- this.mmf = MemoryMappedFile.CreateFromFile(filePath, FileMode.Open,
- null, 0L, MemoryMappedFileAccess.Read);
- this.Open(this.mmf, filePath);
- }
+ this.FileName = fileName;
- public void Open(MemoryMappedFile mmf, string filePath = null)
- {
- this.FileName = filePath;
+ this.mmf = MemoryMappedFile.CreateFromFile(fileName, FileMode.Open, null, 0L, MemoryMappedFileAccess.Read);
+ this.mmva = mmf.CreateViewAccessor(0L, 0L, MemoryMappedFileAccess.Read);
- using (MemoryMappedViewStream stream = mmf.CreateViewStream(
- 0L, 0L, MemoryMappedFileAccess.Read))
- using (BinaryReader reader = new BinaryReader(stream))
+ byte* ptr = null;
+ this.mmva.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
+
+ using (var stream = this.mmf.CreateViewStream(0L, 0L, MemoryMappedFileAccess.Read))
+ using (var reader = new BinaryReader(stream))
{
uint magic = reader.ReadUInt32();
- if (stream.CanSeek && stream.Length < (magic ^ DictionaryMagicID)) //正確なサイズ取得ができないので不等号で代用
- throw new MeCabInvalidFileException("dictionary file is broken", filePath);
+ if (this.mmva.Capacity < (magic ^ DictionaryMagicID))
+ throw new MeCabInvalidFileException("dictionary file is broken", fileName);
this.Version = reader.ReadUInt32();
if (this.Version != DicVersion)
- throw new MeCabInvalidFileException("incompatible version", filePath);
+ throw new MeCabInvalidFileException("incompatible version", fileName);
this.Type = (DictionaryType)reader.ReadUInt32();
this.LexSize = reader.ReadUInt32();
string charSet = StrUtils.GetString(reader.ReadBytes(32), Encoding.ASCII);
this.encoding = StrUtils.GetEncoding(charSet);
- long offset = stream.Position;
- this.da.Open(mmf, offset, dSize);
- offset += dSize;
- this.tokens = mmf.CreateViewAccessor(offset, tSize, MemoryMappedFileAccess.Read);
- offset += tSize;
- this.features = mmf.CreateViewAccessor(offset, fSize, MemoryMappedFileAccess.Read);
+ ptr += stream.Position;
+
+ this.da.Open(ptr, (int)dSize);
+ ptr += dSize;
+
+ this.tokens = (Token*)ptr;
+ ptr += tSize;
+
+ this.features = ptr;
}
}
#else
- public void Open(string filePath)
+ public void Open(string fileName)
{
- this.FileName = filePath;
-
- using (FileStream fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
+ this.FileName = fileName;
+
+ using (FileStream fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read))
using (BinaryReader reader = new BinaryReader(fileStream))
{
this.Open(reader);
string charSet = StrUtils.GetString(reader.ReadBytes(32), Encoding.ASCII);
this.encoding = StrUtils.GetEncoding(charSet);
- this.da.Open(reader, dSize);
+ this.da.Open(reader, (int)dSize);
this.tokens = new Token[tSize / sizeof(Token)];
for (int i = 0; i < this.tokens.Length; i++)
byte* bytes = stackalloc byte[maxByteCount];
int bytesLen = this.encoding.GetBytes(key, len, bytes, maxByteCount);
- DoubleArray.ResultPair result = this.da.ExactMatchSearch(bytes, bytesLen, nodePos);
+ var result = this.da.ExactMatchSearch(bytes, bytesLen, nodePos);
//文字数をデコードしたものに変換
result.Length = this.encoding.GetCharCount(bytes, result.Length);
#region Get Infomation
- public unsafe Token[] GetToken(DoubleArray.ResultPair n)
+ public int GetTokenSize(int value)
{
- Token[] dist = new Token[0xFF & n.Value];
- int tokenPos = n.Value >> 8;
+ return 0xFF & value;
+ }
+
+ public int GetTokenPos(int value)
+ {
+ return value >> 8;
+ }
+
#if MMF_DIC
- this.tokens.ReadArray<Token>(tokenPos * sizeof(Token), dist, 0, dist.Length);
+ public unsafe Token* GetTokens(int value)
+ {
+ return this.tokens + this.GetTokenPos(value);
+ }
+
+ public unsafe Token[] GetTokensArray(int value)
+ {
+ var ret = new Token[this.GetTokenSize(value)];
+ var t = this.GetTokens(value);
+
+ for (int i = 0; i < ret.Length; i++)
+ {
+ ret[i] = t[i];
+ }
+
+ return ret;
+ }
#else
- Array.Copy(this.tokens, tokenPos, dist, 0, dist.Length);
-#endif
- return dist;
+ public ArraySegment<Token> GetTokens(int value)
+ {
+ return new ArraySegment<Token>(this.tokens, this.GetTokenPos(value), this.GetTokenSize(value));
+ }
+
+ public Token[] GetTokensArray(int value)
+ {
+ var ret = new Token[this.GetTokenSize(value)];
+ Array.Copy(this.tokens, this.GetTokenPos(value), ret, 0, ret.Length);
+ return ret;
}
+#endif
- public string GetFeature(uint featurePos)
+ public unsafe string GetFeature(uint featurePos)
{
return StrUtils.GetString(this.features, (long)featurePos, this.encoding);
}
if (disposing)
{
- if (this.da != null) this.da.Dispose();
#if MMF_DIC
- if (this.mmf != null) this.mmf.Dispose();
- if (this.tokens != null) this.tokens.Dispose();
- if (this.features != null) this.features.Dispose();
+ if (this.mmva != null)
+ {
+ this.mmva.SafeMemoryMappedViewHandle.ReleasePointer();
+ this.mmva.Dispose();
+ }
+
+ if (this.mmf != null)
+ this.mmf.Dispose();
#endif
}
public unsafe static string GetString(byte[] bytes, long offset, Encoding enc)
{
fixed (byte* pBytes = bytes)
- return StrUtils.GetString(pBytes + offset, enc);
+ return StrUtils.GetString(pBytes, offset, enc);
}
/// <summary>
/// <remarks>
/// バイト配列の長さはInt32.MaxValueを超えていても良い。
/// </remarks>
+ /// <param name="bytes">デコードするバイトへのポインタ</param>
+ /// <param name="offset">オフセット位置</param>
+ /// <param name="enc">文字エンコーディング</param>
+ /// <returns>文字列（\0は含まない）</returns>
+ public unsafe static string GetString(byte* bytes, long offset, Encoding enc)
+ {
+ return StrUtils.GetString(bytes + offset, enc);
+ }
+ /// <summary>
+ /// バイト配列の中から終端が\0で表された文字列を取り出す。
+ /// </summary>
+ /// <remarks>
+ /// バイト配列の長さはInt32.MaxValueを超えていても良い。
+ /// </remarks>
/// <param name="bytes">デコードする最初のバイトへのポインタ</param>
/// <param name="enc">文字エンコーディング</param>
/// <returns>文字列（\0は含まない）</returns>
{
//バイト長のカウント
int byteCount = 0;
- while (*bytes != Nul) //終端\0に到達するまでシーク
+ while (bytes[byteCount] != Nul) //終端\0に到達するまでシーク
{
checked { byteCount++; } //文字列のバイト長がInt32.MaxValueを超えたならエラー
- bytes++;
}
- bytes -= byteCount;
+
+ if (byteCount == 0)
+ return "";
//生成されうる最大文字数のバッファを確保
int maxCharCount = enc.GetMaxCharCount(byteCount);
}
}
-#if MMF_DIC
-
- /// <summary>
- /// MemoryMappedViewAccessorから終端が\0で表された文字列を取り出す。
- /// </summary>
- /// <remarks>
- /// MemoryMappedViewAccessorの容量はInt32.MaxValueを超えていても良い。
- /// </remarks>
- /// <param name="accessor">MemoryMappedViewAccessor</param>
- /// <param name="index">オフセット位置</param>
- /// <param name="enc">文字エンコーディング</param>
- /// <param name="buffSize">内部で使用するバッファの初期サイズ</param>
- /// <returns>文字列（\0は含まない）</returns>
- public static string GetString(MemoryMappedViewAccessor accessor, long offset, Encoding enc,
- int buffSize = 128)
- {
- byte[] buff = new byte[buffSize]; //IO回数削減のためのバッファ配列
- accessor.ReadArray<byte>(offset, buff, 0, buffSize); //初期読込
-
- //バイト長のカウント
- int byteCount = 0;
- while (buff[byteCount] != Nul) //終端\0に到達するまでシーク
- {
- byteCount++;
-
- if (byteCount == buffSize) //バッファ配列の終端
- {
- //バッファ配列の拡張と追加読込
- checked { buffSize *= 2; } //Int32.MaxValueを超えたならエラー
- byte[] newBuff = new byte[buffSize];
- Buffer.BlockCopy(buff, 0, newBuff, 0, byteCount);
- accessor.ReadArray<byte>(offset + byteCount, newBuff, byteCount, buffSize - byteCount);
- buff = newBuff;
- }
- }
-
- //バッファ配列を文字列にデコード
- return enc.GetString(buff, 0, byteCount);
- }
-
-#endif
-
/// <summary>
/// 指定の名前に対応するエンコーディングを取得する（.NET FWが対応していない名前にもアドホックに対応）
/// </summary>
/// <summary>\r
/// 右文脈 id\r
/// </summary>\r
- public ushort LcAttr { get; set; }\r
+ public ushort LcAttr;\r
\r
/// <summary>\r
/// 左文脈 id\r
/// </summary>\r
- public ushort RcAttr { get; set; }\r
+ public ushort RcAttr;\r
\r
/// <summary>\r
/// 形態素 ID\r
/// </summary>\r
- public ushort PosId { get; set; }\r
+ public ushort PosId;\r
\r
/// <summary>\r
/// 単語生起コスト\r
/// </summary>\r
- public short WCost { get; set; }\r
+ public short WCost;\r
\r
/// <summary>\r
/// 素性情報の位置\r
/// </summary>\r
- public uint Feature { get; set; }\r
+ public uint Feature;\r
\r
/// <summary>\r
/// reserved for noun compound\r
/// </summary>\r
- public uint Compound { get; set; }\r
+ public uint Compound;\r
\r
#endregion\r
\r
if (this.unkDic.Type != DictionaryType.Unk)
throw new MeCabInvalidFileException("not a unk dictionary", this.unkDic.FileName);
- MeCabDictionary sysDic = new MeCabDictionary();
+ var sysDic = new MeCabDictionary();
sysDic.Open(Path.Combine(prefix, SysDicFile));
if (sysDic.Type != DictionaryType.Sys)
throw new MeCabInvalidFileException("not a system dictionary", sysDic.FileName);
for (int i = 0; i < param.UserDic.Length; i++)
{
- MeCabDictionary d = new MeCabDictionary();
+ var d = new MeCabDictionary();
d.Open(Path.Combine(prefix, param.UserDic[i]));
if (d.Type != DictionaryType.Usr)
throw new MeCabInvalidFileException("not a user dictionary", d.FileName);
for (int i = 0; i < this.unkTokens.Length; i++)
{
string key = this.property.Name(i);
- DoubleArray.ResultPair n = this.unkDic.ExactMatchSearch(key);
+ var n = this.unkDic.ExactMatchSearch(key);
if (n.Value == -1)
throw new MeCabInvalidFileException("cannot find UNK category: " + key, this.unkDic.FileName);
- this.unkTokens[i] = this.unkDic.GetToken(n);
+
+ this.unkTokens[i] = this.unkDic.GetTokensArray(n.Value);
}
this.space = this.property.GetCharInfo(' ');
if (end - begin > ushort.MaxValue) end = begin + ushort.MaxValue;
char* begin2 = property.SeekToOtherType(begin, end, this.space, &cInfo, &cLen);
- DoubleArray.ResultPair* daResults = stackalloc DoubleArray.ResultPair[DAResultSize];
+ var daResults = stackalloc DoubleArray.ResultPair[DAResultSize];
foreach (MeCabDictionary it in this.dic)
{
int n = it.CommonPrefixSearch(begin2, (int)(end - begin2), daResults, DAResultSize);
-
for (int i = 0; i < n; i++)
{
- Token[] token = it.GetToken(daResults[i]);
- for (int j = 0; j < token.Length; j++)
+#if MMF_DIC
+ var tokenSize = it.GetTokenSize(daResults->Value);
+ var tokens = it.GetTokens(daResults->Value);
+ for (int j = 0; j < tokenSize; j++)
+#else
+ var seg = it.GetTokens(daResults->Value);
+ var tokens = seg.Array;
+ for (int j = seg.Offset; j < seg.Offset + seg.Count; j++)
+#endif
{
- MeCabNode newNode = this.GetNewNode();
- this.ReadNodeInfo(it, token[j], newNode);
- //newNode.Token = token[j];
- newNode.Length = daResults[i].Length;
- newNode.RLength = (int)(begin2 - begin) + daResults[i].Length;
- newNode.Surface = new string(begin2, 0, daResults[i].Length);
+ var newNode = this.GetNewNode();
+ this.ReadNodeInfo(it, tokens[j], newNode);
+ newNode.Length = daResults->Length;
+ newNode.RLength = (int)(begin2 - begin) + daResults->Length;
+ newNode.Surface = new string(begin2, 0, newNode.Length);
newNode.Stat = MeCabNodeStat.Nor;
newNode.CharType = cInfo.DefaultType;
newNode.BNext = resultNode;
resultNode = newNode;
}
+ daResults++;
}
}
private unsafe void AddUnknown(ref MeCabNode resultNode, CharInfo cInfo,
char* begin, char* begin2, char* begin3)
{
- Token[] token = this.unkTokens[cInfo.DefaultType];
+ var token = this.unkTokens[cInfo.DefaultType];
for (int i = 0; i < token.Length; i++)
{
- MeCabNode newNode = this.GetNewNode();
+ var newNode = this.GetNewNode();
this.ReadNodeInfo(this.unkDic, token[i], newNode);
- newNode.CharType = cInfo.DefaultType;
- newNode.Surface = new string(begin2, 0, (int)(begin3 - begin2));
newNode.Length = (int)(begin3 - begin2);
newNode.RLength = (int)(begin3 - begin);
- newNode.BNext = resultNode;
+ newNode.Surface = new string(begin2, 0, newNode.Length);
+ newNode.CharType = cInfo.DefaultType;
newNode.Stat = MeCabNodeStat.Unk;
if (this.unkFeature != null) newNode.Feature = this.unkFeature;
+ newNode.BNext = resultNode;
resultNode = newNode;
}
}
public MeCabNode GetBosNode()
{
- MeCabNode bosNode = this.GetNewNode();
+ var bosNode = this.GetNewNode();
bosNode.Surface = BosKey; // dummy
bosNode.Feature = this.bosFeature;
bosNode.IsBest = true;
public MeCabNode GetEosNode()
{
- MeCabNode eosNode = this.GetBosNode(); // same
+ var eosNode = this.GetBosNode(); // same
eosNode.Stat = MeCabNodeStat.Eos;
return eosNode;
}
public MeCabNode GetNewNode()
{
- MeCabNode node = new MeCabNode();
+ var node = new MeCabNode();
#if NeedId
node.Id = Tokenizer.id++;
#endif
if (disposing)
{
if (this.dic != null)
- foreach (MeCabDictionary d in this.dic)
- if (d != null) d.Dispose();
+ foreach (var d in this.dic)
+ if (d != null)
+ d.Dispose();
- if (this.unkDic != null) this.unkDic.Dispose();
+ if (this.unkDic != null)
+ this.unkDic.Dispose();
}
this.disposed = true;
OSDN Git Service
