1 #include "io/tokenizer.h"
4 * @brief 各種データテキストをトークン単位に分解する / Extract the first few "tokens" from a buffer
5 * @param buf データテキストの参照ポインタ
7 * @param tokens トークンを保管する文字列参照ポインタ配列
12 * This function uses "colon" and "slash" as the delimeter characters.
13 * We never extract more than "num" tokens. The "last" token may include
14 * "delimeter" characters, allowing the buffer to include a "string" token.
15 * We save pointers to the tokens in "tokens", and return the number found.
16 * Hack -- Attempt to handle the 'c' character formalism
17 * Hack -- An empty buffer, or a final delimeter, yields an "empty" token.
18 * Hack -- We will always extract at least one token
21 s16b tokenize(char *buf, s16b num, char **tokens, BIT_FLAGS mode)
30 if ((*t == ':') || (*t == '/')) break;
32 if ((mode & TOKENIZE_CHECKQUOTE) && (*t == '\''))
39 if (*t != '\'') *t = '\'';