From ad322d83c7f53abd684a261086e81c175fdf5726 Mon Sep 17 00:00:00 2001
From: Hourier <hourier@users.sourceforge.jp>
Date: Sun, 1 Mar 2020 17:57:32 +0900
Subject: [PATCH] [Refactor] #39962 Separated tokenizer.c/h from files.c

---
 Hengband_vcs2017/Hengband/Hengband.vcxproj         |  2 +
 Hengband_vcs2017/Hengband/Hengband.vcxproj.filters |  6 +++
 src/Makefile.am                                    |  1 +
 src/dungeon-file.c                                 |  1 +
 src/files.c                                        | 55 +---------------------
 src/files.h                                        |  2 -
 src/io/tokenizer.c                                 | 54 +++++++++++++++++++++
 src/io/tokenizer.h                                 |  7 +++
 src/rumor.c                                        |  1 +
 src/wild.c                                         |  1 +
 10 files changed, 74 insertions(+), 56 deletions(-)
 create mode 100644 src/io/tokenizer.c
 create mode 100644 src/io/tokenizer.h

diff --git a/Hengband_vcs2017/Hengband/Hengband.vcxproj b/Hengband_vcs2017/Hengband/Hengband.vcxproj
index 1b17d30ec..1c201f177 100644
--- a/Hengband_vcs2017/Hengband/Hengband.vcxproj
+++ b/Hengband_vcs2017/Hengband/Hengband.vcxproj
@@ -191,6 +191,7 @@
     <ClCompile Include="..\..\src\horror-descriptions.c" />
     <ClCompile Include="..\..\src\init.c" />
     <ClCompile Include="..\..\src\io\gf-descriptions.c" />
+    <ClCompile Include="..\..\src\io\tokenizer.c" />
     <ClCompile Include="..\..\src\monster-dist-offsets.c" />
     <ClCompile Include="..\..\src\monster-process.c" />
     <ClCompile Include="..\..\src\monster-status.c" />
@@ -332,6 +333,7 @@
     <ClInclude Include="..\..\src\english.h" />
     <ClInclude Include="..\..\src\horror-descriptions.h" />
     <ClInclude Include="..\..\src\io\gf-descriptions.h" />
+    <ClInclude Include="..\..\src\io\tokenizer.h" />
     <ClInclude Include="..\..\src\monster-dist-offsets.h" />
     <ClInclude Include="..\..\src\object-flavor.h" />
     <ClInclude Include="..\..\src\player-inventory.h" />
diff --git a/Hengband_vcs2017/Hengband/Hengband.vcxproj.filters b/Hengband_vcs2017/Hengband/Hengband.vcxproj.filters
index f225ee9cb..8b21f8f43 100644
--- a/Hengband_vcs2017/Hengband/Hengband.vcxproj.filters
+++ b/Hengband_vcs2017/Hengband/Hengband.vcxproj.filters
@@ -451,6 +451,9 @@
     <ClCompile Include="..\..\src\view\display-player-middle.c">
       <Filter>view</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\io\tokenizer.c">
+      <Filter>io</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\src\gamevalue.h" />
@@ -884,6 +887,9 @@
     <ClInclude Include="..\..\src\view\display-player-middle.h">
       <Filter>view</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\io\tokenizer.h">
+      <Filter>io</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\..\src\wall.bmp" />
diff --git a/src/Makefile.am b/src/Makefile.am
index ae265b764..ad1f96e7d 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -27,6 +27,7 @@ hengband_SOURCES = \
 	player/temporary-resistances.c player/temporary-resistances.h \
 	\
 	io/gf-descriptions.c io/gf-descriptions.h \
+	io/tokenizer.c io/tokenizer.h \
 	signal-handlers.c signal-handlers.h uid-checker.c uid-checker.h \
 	character-dump.c character-dump.h core.c core.h files.c files.h \
 	\
diff --git a/src/dungeon-file.c b/src/dungeon-file.c
index 018e2c2b4..bede3767a 100644
--- a/src/dungeon-file.c
+++ b/src/dungeon-file.c
@@ -35,6 +35,7 @@
 #include "util.h"
 #include "core.h"
 
+#include "io/tokenizer.h"
 #include "files.h"
 #include "dungeon-file.h"
 #include "rooms-vault.h"
diff --git a/src/files.c b/src/files.c
index b465e80c2..ca5e73064 100644
--- a/src/files.c
+++ b/src/files.c
@@ -32,6 +32,7 @@
 #include "autopick.h"
 #include "save.h"
 #include "io/gf-descriptions.h"
+#include "io/tokenizer.h"
 
 #define PREF_TYPE_NORMAL   0
 #define PREF_TYPE_AUTOPICK 1
@@ -59,60 +60,6 @@ char savefile[1024];
 char savefile_base[40];
 
 /*!
- * @brief 各種データテキストをトークン単位に分解する / Extract the first few "tokens" from a buffer
- * @param buf データテキストの参照ポインタ
- * @param num トークンの数
- * @param tokens トークンを保管する文字列参照ポインタ配列
- * @param mode オプション
- * @return 解釈した文字列数
- * @details
- * <pre>
- * This function uses "colon" and "slash" as the delimeter characters.
- * We never extract more than "num" tokens.  The "last" token may include
- * "delimeter" characters, allowing the buffer to include a "string" token.
- * We save pointers to the tokens in "tokens", and return the number found.
- * Hack -- Attempt to handle the 'c' character formalism
- * Hack -- An empty buffer, or a final delimeter, yields an "empty" token.
- * Hack -- We will always extract at least one token
- * </pre>
- */
-s16b tokenize(char *buf, s16b num, char **tokens, BIT_FLAGS mode)
-{
-	s16b i = 0;
-	char *s = buf;
-	while (i < num - 1)
-	{
-		char *t;
-		for (t = s; *t; t++)
-		{
-			if ((*t == ':') || (*t == '/')) break;
-
-			if ((mode & TOKENIZE_CHECKQUOTE) && (*t == '\''))
-			{
-				t++;
-				if (*t == '\\') t++;
-				if (!*t) break;
-
-				t++;
-				if (*t != '\'') *t = '\'';
-			}
-
-			if (*t == '\\') t++;
-		}
-
-		if (!*t) break;
-
-		*t++ = '\0';
-		tokens[i++] = s;
-		s = t;
-	}
-
-	tokens[i++] = s;
-	return i;
-}
-
-
-/*!
  * @brief 設定ファイルの各行から各種テキスト情報を取得する /
  * Parse a sub-file of the "extra info" (format shown below)
  * @param creature_ptr プレーヤーへの参照ポインタ
diff --git a/src/files.h b/src/files.h
index ae49ecd81..618e15f65 100644
--- a/src/files.h
+++ b/src/files.h
@@ -1,7 +1,5 @@
 #pragma once
 
-#define TOKENIZE_CHECKQUOTE 0x01  /* Special handling of single quotes */
-
 extern char savefile[1024];
 extern char savefile_base[40];
 
diff --git a/src/io/tokenizer.c b/src/io/tokenizer.c
new file mode 100644
index 000000000..a884c661a
--- /dev/null
+++ b/src/io/tokenizer.c
@@ -0,0 +1,54 @@
+#include "io/tokenizer.h"
+
+/*!
+ * @brief 各種データテキストをトークン単位に分解する / Extract the first few "tokens" from a buffer
+ * @param buf データテキストの参照ポインタ
+ * @param num トークンの数
+ * @param tokens トークンを保管する文字列参照ポインタ配列
+ * @param mode オプション
+ * @return 解釈した文字列数
+ * @details
+ * <pre>
+ * This function uses "colon" and "slash" as the delimeter characters.
+ * We never extract more than "num" tokens.  The "last" token may include
+ * "delimeter" characters, allowing the buffer to include a "string" token.
+ * We save pointers to the tokens in "tokens", and return the number found.
+ * Hack -- Attempt to handle the 'c' character formalism
+ * Hack -- An empty buffer, or a final delimeter, yields an "empty" token.
+ * Hack -- We will always extract at least one token
+ * </pre>
+ */
+s16b tokenize(char *buf, s16b num, char **tokens, BIT_FLAGS mode)
+{
+	s16b i = 0;
+	char *s = buf;
+	while (i < num - 1)
+	{
+		char *t;
+		for (t = s; *t; t++)
+		{
+			if ((*t == ':') || (*t == '/')) break;
+
+			if ((mode & TOKENIZE_CHECKQUOTE) && (*t == '\''))
+			{
+				t++;
+				if (*t == '\\') t++;
+				if (!*t) break;
+
+				t++;
+				if (*t != '\'') *t = '\'';
+			}
+
+			if (*t == '\\') t++;
+		}
+
+		if (!*t) break;
+
+		*t++ = '\0';
+		tokens[i++] = s;
+		s = t;
+	}
+
+	tokens[i++] = s;
+	return i;
+}
diff --git a/src/io/tokenizer.h b/src/io/tokenizer.h
new file mode 100644
index 000000000..9c4812463
--- /dev/null
+++ b/src/io/tokenizer.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include "angband.h"
+
+#define TOKENIZE_CHECKQUOTE 0x01  /* Special handling of single quotes */
+
+s16b tokenize(char *buf, s16b num, char **tokens, BIT_FLAGS mode);
diff --git a/src/rumor.c b/src/rumor.c
index 7c35c9317..544e373e1 100644
--- a/src/rumor.c
+++ b/src/rumor.c
@@ -1,6 +1,7 @@
 #include "angband.h"
 #include "util.h"
 
+#include "io/tokenizer.h"
 #include "files.h"
 #include "object-flavor.h"
 #include "artifact.h"
diff --git a/src/wild.c b/src/wild.c
index 25e984b91..9ff782904 100644
--- a/src/wild.c
+++ b/src/wild.c
@@ -26,6 +26,7 @@
 #include "monster-status.h"
 #include "quest.h"
 #include "dungeon-file.h"
+#include "io/tokenizer.h"
 #include "files.h"
 #include "feature.h"
 #include "floor-town.h"
-- 
2.11.0