vendor/github.com/pelletier/go-toml/lexer_test.go

   1 package toml
   2
   3 import (
   4         "reflect"
   5         "testing"
   6 )
   7
   8 func testFlow(t *testing.T, input string, expectedFlow []token) {
   9         tokens := lexToml([]byte(input))
  10         if !reflect.DeepEqual(tokens, expectedFlow) {
  11                 t.Fatal("Different flows. Expected\n", expectedFlow, "\nGot:\n", tokens)
  12         }
  13 }
  14
  15 func TestValidKeyGroup(t *testing.T) {
  16         testFlow(t, "[hello world]", []token{
  17                 {Position{1, 1}, tokenLeftBracket, "["},
  18                 {Position{1, 2}, tokenKeyGroup, "hello world"},
  19                 {Position{1, 13}, tokenRightBracket, "]"},
  20                 {Position{1, 14}, tokenEOF, ""},
  21         })
  22 }
  23
  24 func TestNestedQuotedUnicodeKeyGroup(t *testing.T) {
  25         testFlow(t, `[ j . "ʞ" . l ]`, []token{
  26                 {Position{1, 1}, tokenLeftBracket, "["},
  27                 {Position{1, 2}, tokenKeyGroup, ` j . "ʞ" . l `},
  28                 {Position{1, 15}, tokenRightBracket, "]"},
  29                 {Position{1, 16}, tokenEOF, ""},
  30         })
  31 }
  32
  33 func TestUnclosedKeyGroup(t *testing.T) {
  34         testFlow(t, "[hello world", []token{
  35                 {Position{1, 1}, tokenLeftBracket, "["},
  36                 {Position{1, 2}, tokenError, "unclosed table key"},
  37         })
  38 }
  39
  40 func TestComment(t *testing.T) {
  41         testFlow(t, "# blahblah", []token{
  42                 {Position{1, 11}, tokenEOF, ""},
  43         })
  44 }
  45
  46 func TestKeyGroupComment(t *testing.T) {
  47         testFlow(t, "[hello world] # blahblah", []token{
  48                 {Position{1, 1}, tokenLeftBracket, "["},
  49                 {Position{1, 2}, tokenKeyGroup, "hello world"},
  50                 {Position{1, 13}, tokenRightBracket, "]"},
  51                 {Position{1, 25}, tokenEOF, ""},
  52         })
  53 }
  54
  55 func TestMultipleKeyGroupsComment(t *testing.T) {
  56         testFlow(t, "[hello world] # blahblah\n[test]", []token{
  57                 {Position{1, 1}, tokenLeftBracket, "["},
  58                 {Position{1, 2}, tokenKeyGroup, "hello world"},
  59                 {Position{1, 13}, tokenRightBracket, "]"},
  60                 {Position{2, 1}, tokenLeftBracket, "["},
  61                 {Position{2, 2}, tokenKeyGroup, "test"},
  62                 {Position{2, 6}, tokenRightBracket, "]"},
  63                 {Position{2, 7}, tokenEOF, ""},
  64         })
  65 }
  66
  67 func TestSimpleWindowsCRLF(t *testing.T) {
  68         testFlow(t, "a=4\r\nb=2", []token{
  69                 {Position{1, 1}, tokenKey, "a"},
  70                 {Position{1, 2}, tokenEqual, "="},
  71                 {Position{1, 3}, tokenInteger, "4"},
  72                 {Position{2, 1}, tokenKey, "b"},
  73                 {Position{2, 2}, tokenEqual, "="},
  74                 {Position{2, 3}, tokenInteger, "2"},
  75                 {Position{2, 4}, tokenEOF, ""},
  76         })
  77 }
  78
  79 func TestBasicKey(t *testing.T) {
  80         testFlow(t, "hello", []token{
  81                 {Position{1, 1}, tokenKey, "hello"},
  82                 {Position{1, 6}, tokenEOF, ""},
  83         })
  84 }
  85
  86 func TestBasicKeyWithUnderscore(t *testing.T) {
  87         testFlow(t, "hello_hello", []token{
  88                 {Position{1, 1}, tokenKey, "hello_hello"},
  89                 {Position{1, 12}, tokenEOF, ""},
  90         })
  91 }
  92
  93 func TestBasicKeyWithDash(t *testing.T) {
  94         testFlow(t, "hello-world", []token{
  95                 {Position{1, 1}, tokenKey, "hello-world"},
  96                 {Position{1, 12}, tokenEOF, ""},
  97         })
  98 }
  99
 100 func TestBasicKeyWithUppercaseMix(t *testing.T) {
 101         testFlow(t, "helloHELLOHello", []token{
 102                 {Position{1, 1}, tokenKey, "helloHELLOHello"},
 103                 {Position{1, 16}, tokenEOF, ""},
 104         })
 105 }
 106
 107 func TestBasicKeyWithInternationalCharacters(t *testing.T) {
 108         testFlow(t, "héllÖ", []token{
 109                 {Position{1, 1}, tokenKey, "héllÖ"},
 110                 {Position{1, 6}, tokenEOF, ""},
 111         })
 112 }
 113
 114 func TestBasicKeyAndEqual(t *testing.T) {
 115         testFlow(t, "hello =", []token{
 116                 {Position{1, 1}, tokenKey, "hello"},
 117                 {Position{1, 7}, tokenEqual, "="},
 118                 {Position{1, 8}, tokenEOF, ""},
 119         })
 120 }
 121
 122 func TestKeyWithSharpAndEqual(t *testing.T) {
 123         testFlow(t, "key#name = 5", []token{
 124                 {Position{1, 1}, tokenError, "keys cannot contain # character"},
 125         })
 126 }
 127
 128 func TestKeyWithSymbolsAndEqual(t *testing.T) {
 129         testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
 130                 {Position{1, 1}, tokenError, "keys cannot contain ~ character"},
 131         })
 132 }
 133
 134 func TestKeyEqualStringEscape(t *testing.T) {
 135         testFlow(t, `foo = "hello\""`, []token{
 136                 {Position{1, 1}, tokenKey, "foo"},
 137                 {Position{1, 5}, tokenEqual, "="},
 138                 {Position{1, 8}, tokenString, "hello\""},
 139                 {Position{1, 16}, tokenEOF, ""},
 140         })
 141 }
 142
 143 func TestKeyEqualStringUnfinished(t *testing.T) {
 144         testFlow(t, `foo = "bar`, []token{
 145                 {Position{1, 1}, tokenKey, "foo"},
 146                 {Position{1, 5}, tokenEqual, "="},
 147                 {Position{1, 8}, tokenError, "unclosed string"},
 148         })
 149 }
 150
 151 func TestKeyEqualString(t *testing.T) {
 152         testFlow(t, `foo = "bar"`, []token{
 153                 {Position{1, 1}, tokenKey, "foo"},
 154                 {Position{1, 5}, tokenEqual, "="},
 155                 {Position{1, 8}, tokenString, "bar"},
 156                 {Position{1, 12}, tokenEOF, ""},
 157         })
 158 }
 159
 160 func TestKeyEqualTrue(t *testing.T) {
 161         testFlow(t, "foo = true", []token{
 162                 {Position{1, 1}, tokenKey, "foo"},
 163                 {Position{1, 5}, tokenEqual, "="},
 164                 {Position{1, 7}, tokenTrue, "true"},
 165                 {Position{1, 11}, tokenEOF, ""},
 166         })
 167 }
 168
 169 func TestKeyEqualFalse(t *testing.T) {
 170         testFlow(t, "foo = false", []token{
 171                 {Position{1, 1}, tokenKey, "foo"},
 172                 {Position{1, 5}, tokenEqual, "="},
 173                 {Position{1, 7}, tokenFalse, "false"},
 174                 {Position{1, 12}, tokenEOF, ""},
 175         })
 176 }
 177
 178 func TestArrayNestedString(t *testing.T) {
 179         testFlow(t, `a = [ ["hello", "world"] ]`, []token{
 180                 {Position{1, 1}, tokenKey, "a"},
 181                 {Position{1, 3}, tokenEqual, "="},
 182                 {Position{1, 5}, tokenLeftBracket, "["},
 183                 {Position{1, 7}, tokenLeftBracket, "["},
 184                 {Position{1, 9}, tokenString, "hello"},
 185                 {Position{1, 15}, tokenComma, ","},
 186                 {Position{1, 18}, tokenString, "world"},
 187                 {Position{1, 24}, tokenRightBracket, "]"},
 188                 {Position{1, 26}, tokenRightBracket, "]"},
 189                 {Position{1, 27}, tokenEOF, ""},
 190         })
 191 }
 192
 193 func TestArrayNestedInts(t *testing.T) {
 194         testFlow(t, "a = [ [42, 21], [10] ]", []token{
 195                 {Position{1, 1}, tokenKey, "a"},
 196                 {Position{1, 3}, tokenEqual, "="},
 197                 {Position{1, 5}, tokenLeftBracket, "["},
 198                 {Position{1, 7}, tokenLeftBracket, "["},
 199                 {Position{1, 8}, tokenInteger, "42"},
 200                 {Position{1, 10}, tokenComma, ","},
 201                 {Position{1, 12}, tokenInteger, "21"},
 202                 {Position{1, 14}, tokenRightBracket, "]"},
 203                 {Position{1, 15}, tokenComma, ","},
 204                 {Position{1, 17}, tokenLeftBracket, "["},
 205                 {Position{1, 18}, tokenInteger, "10"},
 206                 {Position{1, 20}, tokenRightBracket, "]"},
 207                 {Position{1, 22}, tokenRightBracket, "]"},
 208                 {Position{1, 23}, tokenEOF, ""},
 209         })
 210 }
 211
 212 func TestArrayInts(t *testing.T) {
 213         testFlow(t, "a = [ 42, 21, 10, ]", []token{
 214                 {Position{1, 1}, tokenKey, "a"},
 215                 {Position{1, 3}, tokenEqual, "="},
 216                 {Position{1, 5}, tokenLeftBracket, "["},
 217                 {Position{1, 7}, tokenInteger, "42"},
 218                 {Position{1, 9}, tokenComma, ","},
 219                 {Position{1, 11}, tokenInteger, "21"},
 220                 {Position{1, 13}, tokenComma, ","},
 221                 {Position{1, 15}, tokenInteger, "10"},
 222                 {Position{1, 17}, tokenComma, ","},
 223                 {Position{1, 19}, tokenRightBracket, "]"},
 224                 {Position{1, 20}, tokenEOF, ""},
 225         })
 226 }
 227
 228 func TestMultilineArrayComments(t *testing.T) {
 229         testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
 230                 {Position{1, 1}, tokenKey, "a"},
 231                 {Position{1, 3}, tokenEqual, "="},
 232                 {Position{1, 5}, tokenLeftBracket, "["},
 233                 {Position{1, 6}, tokenInteger, "1"},
 234                 {Position{1, 7}, tokenComma, ","},
 235                 {Position{2, 1}, tokenInteger, "2"},
 236                 {Position{2, 2}, tokenComma, ","},
 237                 {Position{3, 1}, tokenInteger, "3"},
 238                 {Position{3, 2}, tokenComma, ","},
 239                 {Position{4, 1}, tokenRightBracket, "]"},
 240                 {Position{4, 2}, tokenEOF, ""},
 241         })
 242 }
 243
 244 func TestNestedArraysComment(t *testing.T) {
 245         toml := `
 246 someArray = [
 247 # does not work
 248 ["entry1"]
 249 ]`
 250         testFlow(t, toml, []token{
 251                 {Position{2, 1}, tokenKey, "someArray"},
 252                 {Position{2, 11}, tokenEqual, "="},
 253                 {Position{2, 13}, tokenLeftBracket, "["},
 254                 {Position{4, 1}, tokenLeftBracket, "["},
 255                 {Position{4, 3}, tokenString, "entry1"},
 256                 {Position{4, 10}, tokenRightBracket, "]"},
 257                 {Position{5, 1}, tokenRightBracket, "]"},
 258                 {Position{5, 2}, tokenEOF, ""},
 259         })
 260 }
 261
 262 func TestKeyEqualArrayBools(t *testing.T) {
 263         testFlow(t, "foo = [true, false, true]", []token{
 264                 {Position{1, 1}, tokenKey, "foo"},
 265                 {Position{1, 5}, tokenEqual, "="},
 266                 {Position{1, 7}, tokenLeftBracket, "["},
 267                 {Position{1, 8}, tokenTrue, "true"},
 268                 {Position{1, 12}, tokenComma, ","},
 269                 {Position{1, 14}, tokenFalse, "false"},
 270                 {Position{1, 19}, tokenComma, ","},
 271                 {Position{1, 21}, tokenTrue, "true"},
 272                 {Position{1, 25}, tokenRightBracket, "]"},
 273                 {Position{1, 26}, tokenEOF, ""},
 274         })
 275 }
 276
 277 func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
 278         testFlow(t, "foo = [true, false, true] # YEAH", []token{
 279                 {Position{1, 1}, tokenKey, "foo"},
 280                 {Position{1, 5}, tokenEqual, "="},
 281                 {Position{1, 7}, tokenLeftBracket, "["},
 282                 {Position{1, 8}, tokenTrue, "true"},
 283                 {Position{1, 12}, tokenComma, ","},
 284                 {Position{1, 14}, tokenFalse, "false"},
 285                 {Position{1, 19}, tokenComma, ","},
 286                 {Position{1, 21}, tokenTrue, "true"},
 287                 {Position{1, 25}, tokenRightBracket, "]"},
 288                 {Position{1, 33}, tokenEOF, ""},
 289         })
 290 }
 291
 292 func TestDateRegexp(t *testing.T) {
 293         if dateRegexp.FindString("1979-05-27T07:32:00Z") == "" {
 294                 t.Error("basic lexing")
 295         }
 296         if dateRegexp.FindString("1979-05-27T00:32:00-07:00") == "" {
 297                 t.Error("offset lexing")
 298         }
 299         if dateRegexp.FindString("1979-05-27T00:32:00.999999-07:00") == "" {
 300                 t.Error("nano precision lexing")
 301         }
 302 }
 303
 304 func TestKeyEqualDate(t *testing.T) {
 305         testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
 306                 {Position{1, 1}, tokenKey, "foo"},
 307                 {Position{1, 5}, tokenEqual, "="},
 308                 {Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"},
 309                 {Position{1, 27}, tokenEOF, ""},
 310         })
 311         testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{
 312                 {Position{1, 1}, tokenKey, "foo"},
 313                 {Position{1, 5}, tokenEqual, "="},
 314                 {Position{1, 7}, tokenDate, "1979-05-27T00:32:00-07:00"},
 315                 {Position{1, 32}, tokenEOF, ""},
 316         })
 317         testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{
 318                 {Position{1, 1}, tokenKey, "foo"},
 319                 {Position{1, 5}, tokenEqual, "="},
 320                 {Position{1, 7}, tokenDate, "1979-05-27T00:32:00.999999-07:00"},
 321                 {Position{1, 39}, tokenEOF, ""},
 322         })
 323 }
 324
 325 func TestFloatEndingWithDot(t *testing.T) {
 326         testFlow(t, "foo = 42.", []token{
 327                 {Position{1, 1}, tokenKey, "foo"},
 328                 {Position{1, 5}, tokenEqual, "="},
 329                 {Position{1, 7}, tokenError, "float cannot end with a dot"},
 330         })
 331 }
 332
 333 func TestFloatWithTwoDots(t *testing.T) {
 334         testFlow(t, "foo = 4.2.", []token{
 335                 {Position{1, 1}, tokenKey, "foo"},
 336                 {Position{1, 5}, tokenEqual, "="},
 337                 {Position{1, 7}, tokenError, "cannot have two dots in one float"},
 338         })
 339 }
 340
 341 func TestFloatWithExponent1(t *testing.T) {
 342         testFlow(t, "a = 5e+22", []token{
 343                 {Position{1, 1}, tokenKey, "a"},
 344                 {Position{1, 3}, tokenEqual, "="},
 345                 {Position{1, 5}, tokenFloat, "5e+22"},
 346                 {Position{1, 10}, tokenEOF, ""},
 347         })
 348 }
 349
 350 func TestFloatWithExponent2(t *testing.T) {
 351         testFlow(t, "a = 5E+22", []token{
 352                 {Position{1, 1}, tokenKey, "a"},
 353                 {Position{1, 3}, tokenEqual, "="},
 354                 {Position{1, 5}, tokenFloat, "5E+22"},
 355                 {Position{1, 10}, tokenEOF, ""},
 356         })
 357 }
 358
 359 func TestFloatWithExponent3(t *testing.T) {
 360         testFlow(t, "a = -5e+22", []token{
 361                 {Position{1, 1}, tokenKey, "a"},
 362                 {Position{1, 3}, tokenEqual, "="},
 363                 {Position{1, 5}, tokenFloat, "-5e+22"},
 364                 {Position{1, 11}, tokenEOF, ""},
 365         })
 366 }
 367
 368 func TestFloatWithExponent4(t *testing.T) {
 369         testFlow(t, "a = -5e-22", []token{
 370                 {Position{1, 1}, tokenKey, "a"},
 371                 {Position{1, 3}, tokenEqual, "="},
 372                 {Position{1, 5}, tokenFloat, "-5e-22"},
 373                 {Position{1, 11}, tokenEOF, ""},
 374         })
 375 }
 376
 377 func TestFloatWithExponent5(t *testing.T) {
 378         testFlow(t, "a = 6.626e-34", []token{
 379                 {Position{1, 1}, tokenKey, "a"},
 380                 {Position{1, 3}, tokenEqual, "="},
 381                 {Position{1, 5}, tokenFloat, "6.626e-34"},
 382                 {Position{1, 14}, tokenEOF, ""},
 383         })
 384 }
 385
 386 func TestInvalidEsquapeSequence(t *testing.T) {
 387         testFlow(t, `foo = "\x"`, []token{
 388                 {Position{1, 1}, tokenKey, "foo"},
 389                 {Position{1, 5}, tokenEqual, "="},
 390                 {Position{1, 8}, tokenError, "invalid escape sequence: \\x"},
 391         })
 392 }
 393
 394 func TestNestedArrays(t *testing.T) {
 395         testFlow(t, "foo = [[[]]]", []token{
 396                 {Position{1, 1}, tokenKey, "foo"},
 397                 {Position{1, 5}, tokenEqual, "="},
 398                 {Position{1, 7}, tokenLeftBracket, "["},
 399                 {Position{1, 8}, tokenLeftBracket, "["},
 400                 {Position{1, 9}, tokenLeftBracket, "["},
 401                 {Position{1, 10}, tokenRightBracket, "]"},
 402                 {Position{1, 11}, tokenRightBracket, "]"},
 403                 {Position{1, 12}, tokenRightBracket, "]"},
 404                 {Position{1, 13}, tokenEOF, ""},
 405         })
 406 }
 407
 408 func TestKeyEqualNumber(t *testing.T) {
 409         testFlow(t, "foo = 42", []token{
 410                 {Position{1, 1}, tokenKey, "foo"},
 411                 {Position{1, 5}, tokenEqual, "="},
 412                 {Position{1, 7}, tokenInteger, "42"},
 413                 {Position{1, 9}, tokenEOF, ""},
 414         })
 415
 416         testFlow(t, "foo = +42", []token{
 417                 {Position{1, 1}, tokenKey, "foo"},
 418                 {Position{1, 5}, tokenEqual, "="},
 419                 {Position{1, 7}, tokenInteger, "+42"},
 420                 {Position{1, 10}, tokenEOF, ""},
 421         })
 422
 423         testFlow(t, "foo = -42", []token{
 424                 {Position{1, 1}, tokenKey, "foo"},
 425                 {Position{1, 5}, tokenEqual, "="},
 426                 {Position{1, 7}, tokenInteger, "-42"},
 427                 {Position{1, 10}, tokenEOF, ""},
 428         })
 429
 430         testFlow(t, "foo = 4.2", []token{
 431                 {Position{1, 1}, tokenKey, "foo"},
 432                 {Position{1, 5}, tokenEqual, "="},
 433                 {Position{1, 7}, tokenFloat, "4.2"},
 434                 {Position{1, 10}, tokenEOF, ""},
 435         })
 436
 437         testFlow(t, "foo = +4.2", []token{
 438                 {Position{1, 1}, tokenKey, "foo"},
 439                 {Position{1, 5}, tokenEqual, "="},
 440                 {Position{1, 7}, tokenFloat, "+4.2"},
 441                 {Position{1, 11}, tokenEOF, ""},
 442         })
 443
 444         testFlow(t, "foo = -4.2", []token{
 445                 {Position{1, 1}, tokenKey, "foo"},
 446                 {Position{1, 5}, tokenEqual, "="},
 447                 {Position{1, 7}, tokenFloat, "-4.2"},
 448                 {Position{1, 11}, tokenEOF, ""},
 449         })
 450
 451         testFlow(t, "foo = 1_000", []token{
 452                 {Position{1, 1}, tokenKey, "foo"},
 453                 {Position{1, 5}, tokenEqual, "="},
 454                 {Position{1, 7}, tokenInteger, "1_000"},
 455                 {Position{1, 12}, tokenEOF, ""},
 456         })
 457
 458         testFlow(t, "foo = 5_349_221", []token{
 459                 {Position{1, 1}, tokenKey, "foo"},
 460                 {Position{1, 5}, tokenEqual, "="},
 461                 {Position{1, 7}, tokenInteger, "5_349_221"},
 462                 {Position{1, 16}, tokenEOF, ""},
 463         })
 464
 465         testFlow(t, "foo = 1_2_3_4_5", []token{
 466                 {Position{1, 1}, tokenKey, "foo"},
 467                 {Position{1, 5}, tokenEqual, "="},
 468                 {Position{1, 7}, tokenInteger, "1_2_3_4_5"},
 469                 {Position{1, 16}, tokenEOF, ""},
 470         })
 471
 472         testFlow(t, "flt8 = 9_224_617.445_991_228_313", []token{
 473                 {Position{1, 1}, tokenKey, "flt8"},
 474                 {Position{1, 6}, tokenEqual, "="},
 475                 {Position{1, 8}, tokenFloat, "9_224_617.445_991_228_313"},
 476                 {Position{1, 33}, tokenEOF, ""},
 477         })
 478
 479         testFlow(t, "foo = +", []token{
 480                 {Position{1, 1}, tokenKey, "foo"},
 481                 {Position{1, 5}, tokenEqual, "="},
 482                 {Position{1, 7}, tokenError, "no digit in that number"},
 483         })
 484 }
 485
 486 func TestMultiline(t *testing.T) {
 487         testFlow(t, "foo = 42\nbar=21", []token{
 488                 {Position{1, 1}, tokenKey, "foo"},
 489                 {Position{1, 5}, tokenEqual, "="},
 490                 {Position{1, 7}, tokenInteger, "42"},
 491                 {Position{2, 1}, tokenKey, "bar"},
 492                 {Position{2, 4}, tokenEqual, "="},
 493                 {Position{2, 5}, tokenInteger, "21"},
 494                 {Position{2, 7}, tokenEOF, ""},
 495         })
 496 }
 497
 498 func TestKeyEqualStringUnicodeEscape(t *testing.T) {
 499         testFlow(t, `foo = "hello \u2665"`, []token{
 500                 {Position{1, 1}, tokenKey, "foo"},
 501                 {Position{1, 5}, tokenEqual, "="},
 502                 {Position{1, 8}, tokenString, "hello ♥"},
 503                 {Position{1, 21}, tokenEOF, ""},
 504         })
 505         testFlow(t, `foo = "hello \U000003B4"`, []token{
 506                 {Position{1, 1}, tokenKey, "foo"},
 507                 {Position{1, 5}, tokenEqual, "="},
 508                 {Position{1, 8}, tokenString, "hello δ"},
 509                 {Position{1, 25}, tokenEOF, ""},
 510         })
 511         testFlow(t, `foo = "\uabcd"`, []token{
 512                 {Position{1, 1}, tokenKey, "foo"},
 513                 {Position{1, 5}, tokenEqual, "="},
 514                 {Position{1, 8}, tokenString, "\uabcd"},
 515                 {Position{1, 15}, tokenEOF, ""},
 516         })
 517         testFlow(t, `foo = "\uABCD"`, []token{
 518                 {Position{1, 1}, tokenKey, "foo"},
 519                 {Position{1, 5}, tokenEqual, "="},
 520                 {Position{1, 8}, tokenString, "\uABCD"},
 521                 {Position{1, 15}, tokenEOF, ""},
 522         })
 523         testFlow(t, `foo = "\U000bcdef"`, []token{
 524                 {Position{1, 1}, tokenKey, "foo"},
 525                 {Position{1, 5}, tokenEqual, "="},
 526                 {Position{1, 8}, tokenString, "\U000bcdef"},
 527                 {Position{1, 19}, tokenEOF, ""},
 528         })
 529         testFlow(t, `foo = "\U000BCDEF"`, []token{
 530                 {Position{1, 1}, tokenKey, "foo"},
 531                 {Position{1, 5}, tokenEqual, "="},
 532                 {Position{1, 8}, tokenString, "\U000BCDEF"},
 533                 {Position{1, 19}, tokenEOF, ""},
 534         })
 535         testFlow(t, `foo = "\u2"`, []token{
 536                 {Position{1, 1}, tokenKey, "foo"},
 537                 {Position{1, 5}, tokenEqual, "="},
 538                 {Position{1, 8}, tokenError, "unfinished unicode escape"},
 539         })
 540         testFlow(t, `foo = "\U2"`, []token{
 541                 {Position{1, 1}, tokenKey, "foo"},
 542                 {Position{1, 5}, tokenEqual, "="},
 543                 {Position{1, 8}, tokenError, "unfinished unicode escape"},
 544         })
 545 }
 546
 547 func TestKeyEqualStringNoEscape(t *testing.T) {
 548         testFlow(t, "foo = \"hello \u0002\"", []token{
 549                 {Position{1, 1}, tokenKey, "foo"},
 550                 {Position{1, 5}, tokenEqual, "="},
 551                 {Position{1, 8}, tokenError, "unescaped control character U+0002"},
 552         })
 553         testFlow(t, "foo = \"hello \u001F\"", []token{
 554                 {Position{1, 1}, tokenKey, "foo"},
 555                 {Position{1, 5}, tokenEqual, "="},
 556                 {Position{1, 8}, tokenError, "unescaped control character U+001F"},
 557         })
 558 }
 559
 560 func TestLiteralString(t *testing.T) {
 561         testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{
 562                 {Position{1, 1}, tokenKey, "foo"},
 563                 {Position{1, 5}, tokenEqual, "="},
 564                 {Position{1, 8}, tokenString, `C:\Users\nodejs\templates`},
 565                 {Position{1, 34}, tokenEOF, ""},
 566         })
 567         testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{
 568                 {Position{1, 1}, tokenKey, "foo"},
 569                 {Position{1, 5}, tokenEqual, "="},
 570                 {Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`},
 571                 {Position{1, 35}, tokenEOF, ""},
 572         })
 573         testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{
 574                 {Position{1, 1}, tokenKey, "foo"},
 575                 {Position{1, 5}, tokenEqual, "="},
 576                 {Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`},
 577                 {Position{1, 34}, tokenEOF, ""},
 578         })
 579         testFlow(t, `foo = '<\i\c*\s*>'`, []token{
 580                 {Position{1, 1}, tokenKey, "foo"},
 581                 {Position{1, 5}, tokenEqual, "="},
 582                 {Position{1, 8}, tokenString, `<\i\c*\s*>`},
 583                 {Position{1, 19}, tokenEOF, ""},
 584         })
 585         testFlow(t, `foo = 'C:\Users\nodejs\unfinis`, []token{
 586                 {Position{1, 1}, tokenKey, "foo"},
 587                 {Position{1, 5}, tokenEqual, "="},
 588                 {Position{1, 8}, tokenError, "unclosed string"},
 589         })
 590 }
 591
 592 func TestMultilineLiteralString(t *testing.T) {
 593         testFlow(t, `foo = '''hello 'literal' world'''`, []token{
 594                 {Position{1, 1}, tokenKey, "foo"},
 595                 {Position{1, 5}, tokenEqual, "="},
 596                 {Position{1, 10}, tokenString, `hello 'literal' world`},
 597                 {Position{1, 34}, tokenEOF, ""},
 598         })
 599
 600         testFlow(t, "foo = '''\nhello\n'literal'\nworld'''", []token{
 601                 {Position{1, 1}, tokenKey, "foo"},
 602                 {Position{1, 5}, tokenEqual, "="},
 603                 {Position{2, 1}, tokenString, "hello\n'literal'\nworld"},
 604                 {Position{4, 9}, tokenEOF, ""},
 605         })
 606         testFlow(t, "foo = '''\r\nhello\r\n'literal'\r\nworld'''", []token{
 607                 {Position{1, 1}, tokenKey, "foo"},
 608                 {Position{1, 5}, tokenEqual, "="},
 609                 {Position{2, 1}, tokenString, "hello\r\n'literal'\r\nworld"},
 610                 {Position{4, 9}, tokenEOF, ""},
 611         })
 612 }
 613
 614 func TestMultilineString(t *testing.T) {
 615         testFlow(t, `foo = """hello "literal" world"""`, []token{
 616                 {Position{1, 1}, tokenKey, "foo"},
 617                 {Position{1, 5}, tokenEqual, "="},
 618                 {Position{1, 10}, tokenString, `hello "literal" world`},
 619                 {Position{1, 34}, tokenEOF, ""},
 620         })
 621
 622         testFlow(t, "foo = \"\"\"\r\nhello\\\r\n\"literal\"\\\nworld\"\"\"", []token{
 623                 {Position{1, 1}, tokenKey, "foo"},
 624                 {Position{1, 5}, tokenEqual, "="},
 625                 {Position{2, 1}, tokenString, "hello\"literal\"world"},
 626                 {Position{4, 9}, tokenEOF, ""},
 627         })
 628
 629         testFlow(t, "foo = \"\"\"\\\n    \\\n    \\\n    hello\\\nmultiline\\\nworld\"\"\"", []token{
 630                 {Position{1, 1}, tokenKey, "foo"},
 631                 {Position{1, 5}, tokenEqual, "="},
 632                 {Position{1, 10}, tokenString, "hellomultilineworld"},
 633                 {Position{6, 9}, tokenEOF, ""},
 634         })
 635
 636         testFlow(t, "key2 = \"\"\"\nThe quick brown \\\n\n\n  fox jumps over \\\n    the lazy dog.\"\"\"", []token{
 637                 {Position{1, 1}, tokenKey, "key2"},
 638                 {Position{1, 6}, tokenEqual, "="},
 639                 {Position{2, 1}, tokenString, "The quick brown fox jumps over the lazy dog."},
 640                 {Position{6, 21}, tokenEOF, ""},
 641         })
 642
 643         testFlow(t, "key2 = \"\"\"\\\n       The quick brown \\\n       fox jumps over \\\n       the lazy dog.\\\n       \"\"\"", []token{
 644                 {Position{1, 1}, tokenKey, "key2"},
 645                 {Position{1, 6}, tokenEqual, "="},
 646                 {Position{1, 11}, tokenString, "The quick brown fox jumps over the lazy dog."},
 647                 {Position{5, 11}, tokenEOF, ""},
 648         })
 649
 650         testFlow(t, `key2 = "Roses are red\nViolets are blue"`, []token{
 651                 {Position{1, 1}, tokenKey, "key2"},
 652                 {Position{1, 6}, tokenEqual, "="},
 653                 {Position{1, 9}, tokenString, "Roses are red\nViolets are blue"},
 654                 {Position{1, 41}, tokenEOF, ""},
 655         })
 656
 657         testFlow(t, "key2 = \"\"\"\nRoses are red\nViolets are blue\"\"\"", []token{
 658                 {Position{1, 1}, tokenKey, "key2"},
 659                 {Position{1, 6}, tokenEqual, "="},
 660                 {Position{2, 1}, tokenString, "Roses are red\nViolets are blue"},
 661                 {Position{3, 20}, tokenEOF, ""},
 662         })
 663 }
 664
 665 func TestUnicodeString(t *testing.T) {
 666         testFlow(t, `foo = "hello ♥ world"`, []token{
 667                 {Position{1, 1}, tokenKey, "foo"},
 668                 {Position{1, 5}, tokenEqual, "="},
 669                 {Position{1, 8}, tokenString, "hello ♥ world"},
 670                 {Position{1, 22}, tokenEOF, ""},
 671         })
 672 }
 673 func TestEscapeInString(t *testing.T) {
 674         testFlow(t, `foo = "\b\f\/"`, []token{
 675                 {Position{1, 1}, tokenKey, "foo"},
 676                 {Position{1, 5}, tokenEqual, "="},
 677                 {Position{1, 8}, tokenString, "\b\f/"},
 678                 {Position{1, 15}, tokenEOF, ""},
 679         })
 680 }
 681
 682 func TestKeyGroupArray(t *testing.T) {
 683         testFlow(t, "[[foo]]", []token{
 684                 {Position{1, 1}, tokenDoubleLeftBracket, "[["},
 685                 {Position{1, 3}, tokenKeyGroupArray, "foo"},
 686                 {Position{1, 6}, tokenDoubleRightBracket, "]]"},
 687                 {Position{1, 8}, tokenEOF, ""},
 688         })
 689 }
 690
 691 func TestQuotedKey(t *testing.T) {
 692         testFlow(t, "\"a b\" = 42", []token{
 693                 {Position{1, 1}, tokenKey, "\"a b\""},
 694                 {Position{1, 7}, tokenEqual, "="},
 695                 {Position{1, 9}, tokenInteger, "42"},
 696                 {Position{1, 11}, tokenEOF, ""},
 697         })
 698 }
 699
 700 func TestKeyNewline(t *testing.T) {
 701         testFlow(t, "a\n= 4", []token{
 702                 {Position{1, 1}, tokenError, "keys cannot contain new lines"},
 703         })
 704 }
 705
 706 func TestInvalidFloat(t *testing.T) {
 707         testFlow(t, "a=7e1_", []token{
 708                 {Position{1, 1}, tokenKey, "a"},
 709                 {Position{1, 2}, tokenEqual, "="},
 710                 {Position{1, 3}, tokenFloat, "7e1_"},
 711                 {Position{1, 7}, tokenEOF, ""},
 712         })
 713 }
 714
 715 func TestLexUnknownRvalue(t *testing.T) {
 716         testFlow(t, `a = !b`, []token{
 717                 {Position{1, 1}, tokenKey, "a"},
 718                 {Position{1, 3}, tokenEqual, "="},
 719                 {Position{1, 5}, tokenError, "no value can start with !"},
 720         })
 721
 722         testFlow(t, `a = \b`, []token{
 723                 {Position{1, 1}, tokenKey, "a"},
 724                 {Position{1, 3}, tokenEqual, "="},
 725                 {Position{1, 5}, tokenError, `no value can start with \`},
 726         })
 727 }
 728
 729 func BenchmarkLexer(b *testing.B) {
 730         sample := `title = "Hugo: A Fast and Flexible Website Generator"
 731 baseurl = "http://gohugo.io/"
 732 MetaDataFormat = "yaml"
 733 pluralizeListTitles = false
 734
 735 [params]
 736   description = "Documentation of Hugo, a fast and flexible static site generator built with love by spf13, bep and friends in Go"
 737   author = "Steve Francia (spf13) and friends"
 738   release = "0.22-DEV"
 739
 740 [[menu.main]]
 741         name = "Download Hugo"
 742         pre = "<i class='fa fa-download'></i>"
 743         url = "https://github.com/spf13/hugo/releases"
 744         weight = -200
 745 `
 746         b.ResetTimer()
 747         for i := 0; i < b.N; i++ {
 748                 lexToml([]byte(sample))
 749         }
 750 }