1 /** 2 * Defines lexical tokens. 3 * 4 * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens) 5 * 6 * Copyright: Copyright (C) 1999-2023 by The D Language Foundation, All Rights Reserved 7 * Authors: $(LINK2 https://www.digitalmars.com, Walter Bright) 8 * License: $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0) 9 * Source: $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d) 10 * Documentation: https://dlang.org/phobos/dmd_tokens.html 11 * Coverage: https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d 12 */ 13 14 module dmd.tokens; 15 16 import core.stdc.ctype; 17 import core.stdc.stdio; 18 import core.stdc.string; 19 import dmd.identifier; 20 import dmd.location; 21 import dmd.root.ctfloat; 22 import dmd.common.outbuffer; 23 import dmd.root.rmem; 24 import dmd.root.utf; 25 26 enum TOK : ubyte 27 { 28 reserved, 29 30 // Other 31 leftParenthesis, 32 rightParenthesis, 33 leftBracket, 34 rightBracket, 35 leftCurly, 36 rightCurly, 37 colon, 38 semicolon, 39 dotDotDot, 40 endOfFile, 41 cast_, 42 null_, 43 assert_, 44 true_, 45 false_, 46 throw_, 47 new_, 48 delete_, 49 variable, 50 slice, 51 version_, 52 module_, 53 dollar, 54 template_, 55 typeof_, 56 pragma_, 57 typeid_, 58 comment, 59 60 // Operators 61 lessThan, 62 greaterThan, 63 lessOrEqual, 64 greaterOrEqual, 65 equal, 66 notEqual, 67 identity, 68 notIdentity, 69 is_, 70 71 leftShift, 72 rightShift, 73 leftShiftAssign, 74 rightShiftAssign, 75 unsignedRightShift, 76 unsignedRightShiftAssign, 77 concatenateAssign, // ~= 78 add, 79 min, 80 addAssign, 81 minAssign, 82 mul, 83 div, 84 mod, 85 mulAssign, 86 divAssign, 87 modAssign, 88 and, 89 or, 90 xor, 91 andAssign, 92 orAssign, 93 xorAssign, 94 assign, 95 not, 96 tilde, 97 plusPlus, 98 minusMinus, 99 dot, 100 comma, 101 question, 102 andAnd, 103 orOr, 104 105 // Numeric literals 106 int32Literal, 107 uns32Literal, 108 int64Literal, 109 uns64Literal, 110 int128Literal, 111 uns128Literal, 112 float32Literal, 113 float64Literal, 114 float80Literal, 115 imaginary32Literal, 116 imaginary64Literal, 117 imaginary80Literal, 118 119 // Char constants 120 charLiteral, 121 wcharLiteral, 122 dcharLiteral, 123 124 // Leaf operators 125 identifier, 126 string_, 127 this_, 128 super_, 129 error, 130 131 // Basic types 132 void_, 133 int8, 134 uns8, 135 int16, 136 uns16, 137 int32, 138 uns32, 139 int64, 140 uns64, 141 int128, 142 uns128, 143 float32, 144 float64, 145 float80, 146 imaginary32, 147 imaginary64, 148 imaginary80, 149 complex32, 150 complex64, 151 complex80, 152 char_, 153 wchar_, 154 dchar_, 155 bool_, 156 157 // Aggregates 158 struct_, 159 class_, 160 interface_, 161 union_, 162 enum_, 163 import_, 164 alias_, 165 override_, 166 delegate_, 167 function_, 168 mixin_, 169 align_, 170 extern_, 171 private_, 172 protected_, 173 public_, 174 export_, 175 static_, 176 final_, 177 const_, 178 abstract_, 179 debug_, 180 deprecated_, 181 in_, 182 out_, 183 inout_, 184 lazy_, 185 auto_, 186 package_, 187 immutable_, 188 189 // Statements 190 if_, 191 else_, 192 while_, 193 for_, 194 do_, 195 switch_, 196 case_, 197 default_, 198 break_, 199 continue_, 200 with_, 201 synchronized_, 202 return_, 203 goto_, 204 try_, 205 catch_, 206 finally_, 207 asm_, 208 foreach_, 209 foreach_reverse_, 210 scope_, 211 onScopeExit, 212 onScopeFailure, 213 onScopeSuccess, 214 215 // Contracts 216 invariant_, 217 218 // Testing 219 unittest_, 220 221 // Added after 1.0 222 argumentTypes, 223 ref_, 224 macro_, 225 226 parameters, 227 traits, 228 pure_, 229 nothrow_, 230 gshared, 231 line, 232 file, 233 fileFullPath, 234 moduleString, // __MODULE__ 235 functionString, // __FUNCTION__ 236 prettyFunction, // __PRETTY_FUNCTION__ 237 shared_, 238 at, 239 pow, 240 powAssign, 241 goesTo, 242 vector, 243 pound, 244 245 arrow, // -> 246 colonColon, // :: 247 wchar_tLiteral, 248 endOfLine, // \n, \r, \u2028, \u2029 249 whitespace, 250 251 // C only keywords 252 inline, 253 register, 254 restrict, 255 signed, 256 sizeof_, 257 typedef_, 258 unsigned, 259 volatile, 260 _Alignas, 261 _Alignof, 262 _Atomic, 263 _Bool, 264 _Complex, 265 _Generic, 266 _Imaginary, 267 _Noreturn, 268 _Static_assert, 269 _Thread_local, 270 271 // C only extended keywords 272 _assert, 273 _import, 274 __cdecl, 275 __declspec, 276 __stdcall, 277 __thread, 278 __pragma, 279 __int128, 280 __attribute__, 281 } 282 283 /// Expression nodes 284 enum EXP : ubyte 285 { 286 reserved, 287 288 // Other 289 negate, 290 cast_, 291 null_, 292 assert_, 293 array, 294 call, 295 address, 296 type, 297 throw_, 298 new_, 299 delete_, 300 star, 301 symbolOffset, 302 variable, 303 dotVariable, 304 dotIdentifier, 305 dotTemplateInstance, 306 dotType, 307 slice, 308 arrayLength, 309 dollar, 310 template_, 311 dotTemplateDeclaration, 312 declaration, 313 dSymbol, 314 typeid_, 315 uadd, 316 remove, 317 newAnonymousClass, 318 arrayLiteral, 319 assocArrayLiteral, 320 structLiteral, 321 classReference, 322 thrownException, 323 delegatePointer, 324 delegateFunctionPointer, 325 326 // Operators 327 lessThan, 328 greaterThan, 329 lessOrEqual, 330 greaterOrEqual, 331 equal, 332 notEqual, 333 identity, 334 notIdentity, 335 index, 336 is_, 337 338 leftShift, 339 rightShift, 340 leftShiftAssign, 341 rightShiftAssign, 342 unsignedRightShift, 343 unsignedRightShiftAssign, 344 concatenate, 345 concatenateAssign, // ~= 346 concatenateElemAssign, 347 concatenateDcharAssign, 348 add, 349 min, 350 addAssign, 351 minAssign, 352 mul, 353 div, 354 mod, 355 mulAssign, 356 divAssign, 357 modAssign, 358 and, 359 or, 360 xor, 361 andAssign, 362 orAssign, 363 xorAssign, 364 assign, 365 not, 366 tilde, 367 plusPlus, 368 minusMinus, 369 construct, 370 blit, 371 dot, 372 comma, 373 question, 374 andAnd, 375 orOr, 376 prePlusPlus, 377 preMinusMinus, 378 379 // Leaf operators 380 identifier, 381 string_, 382 this_, 383 super_, 384 halt, 385 tuple, 386 error, 387 388 // Basic types 389 void_, 390 int64, 391 float64, 392 complex80, 393 import_, 394 delegate_, 395 function_, 396 mixin_, 397 in_, 398 break_, 399 continue_, 400 goto_, 401 scope_, 402 403 traits, 404 overloadSet, 405 line, 406 file, 407 fileFullPath, 408 moduleString, // __MODULE__ 409 functionString, // __FUNCTION__ 410 prettyFunction, // __PRETTY_FUNCTION__ 411 pow, 412 powAssign, 413 vector, 414 415 voidExpression, 416 cantExpression, 417 showCtfeContext, 418 objcClassReference, 419 vectorArray, 420 compoundLiteral, // ( type-name ) { initializer-list } 421 _Generic, 422 interval, 423 424 loweredAssignExp, 425 } 426 427 enum FirstCKeyword = TOK.inline; 428 429 // Assert that all token enum members have consecutive values and 430 // that none of them overlap 431 static assert(() { 432 foreach (idx, enumName; __traits(allMembers, TOK)) { 433 static if (idx != __traits(getMember, TOK, enumName)) { 434 pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName)); 435 static assert(0); 436 } 437 } 438 return true; 439 }()); 440 441 /**************************************** 442 */ 443 444 private immutable TOK[] keywords = 445 [ 446 TOK.this_, 447 TOK.super_, 448 TOK.assert_, 449 TOK.null_, 450 TOK.true_, 451 TOK.false_, 452 TOK.cast_, 453 TOK.new_, 454 TOK.delete_, 455 TOK.throw_, 456 TOK.module_, 457 TOK.pragma_, 458 TOK.typeof_, 459 TOK.typeid_, 460 TOK.template_, 461 TOK.void_, 462 TOK.int8, 463 TOK.uns8, 464 TOK.int16, 465 TOK.uns16, 466 TOK.int32, 467 TOK.uns32, 468 TOK.int64, 469 TOK.uns64, 470 TOK.int128, 471 TOK.uns128, 472 TOK.float32, 473 TOK.float64, 474 TOK.float80, 475 TOK.bool_, 476 TOK.char_, 477 TOK.wchar_, 478 TOK.dchar_, 479 TOK.imaginary32, 480 TOK.imaginary64, 481 TOK.imaginary80, 482 TOK.complex32, 483 TOK.complex64, 484 TOK.complex80, 485 TOK.delegate_, 486 TOK.function_, 487 TOK.is_, 488 TOK.if_, 489 TOK.else_, 490 TOK.while_, 491 TOK.for_, 492 TOK.do_, 493 TOK.switch_, 494 TOK.case_, 495 TOK.default_, 496 TOK.break_, 497 TOK.continue_, 498 TOK.synchronized_, 499 TOK.return_, 500 TOK.goto_, 501 TOK.try_, 502 TOK.catch_, 503 TOK.finally_, 504 TOK.with_, 505 TOK.asm_, 506 TOK.foreach_, 507 TOK.foreach_reverse_, 508 TOK.scope_, 509 TOK.struct_, 510 TOK.class_, 511 TOK.interface_, 512 TOK.union_, 513 TOK.enum_, 514 TOK.import_, 515 TOK.mixin_, 516 TOK.static_, 517 TOK.final_, 518 TOK.const_, 519 TOK.alias_, 520 TOK.override_, 521 TOK.abstract_, 522 TOK.debug_, 523 TOK.deprecated_, 524 TOK.in_, 525 TOK.out_, 526 TOK.inout_, 527 TOK.lazy_, 528 TOK.auto_, 529 TOK.align_, 530 TOK.extern_, 531 TOK.private_, 532 TOK.package_, 533 TOK.protected_, 534 TOK.public_, 535 TOK.export_, 536 TOK.invariant_, 537 TOK.unittest_, 538 TOK.version_, 539 TOK.argumentTypes, 540 TOK.parameters, 541 TOK.ref_, 542 TOK.macro_, 543 TOK.pure_, 544 TOK.nothrow_, 545 TOK.gshared, 546 TOK.traits, 547 TOK.vector, 548 TOK.file, 549 TOK.fileFullPath, 550 TOK.line, 551 TOK.moduleString, 552 TOK.functionString, 553 TOK.prettyFunction, 554 TOK.shared_, 555 TOK.immutable_, 556 557 // C only keywords 558 TOK.inline, 559 TOK.register, 560 TOK.restrict, 561 TOK.signed, 562 TOK.sizeof_, 563 TOK.typedef_, 564 TOK.unsigned, 565 TOK..volatile, 566 TOK._Alignas, 567 TOK._Alignof, 568 TOK._Atomic, 569 TOK._Bool, 570 TOK._Complex, 571 TOK._Generic, 572 TOK._Imaginary, 573 TOK._Noreturn, 574 TOK._Static_assert, 575 TOK._Thread_local, 576 577 // C only extended keywords 578 TOK._assert, 579 TOK._import, 580 TOK.__cdecl, 581 TOK.__declspec, 582 TOK.__stdcall, 583 TOK.__thread, 584 TOK.__pragma, 585 TOK.__int128, 586 TOK.__attribute__, 587 ]; 588 589 // Initialize the identifier pool 590 shared static this() nothrow 591 { 592 Identifier.initTable(); 593 foreach (kw; keywords) 594 { 595 //printf("keyword[%d] = '%s'\n",kw, Token.tochars[kw].ptr); 596 Identifier.idPool(Token.tochars[kw].ptr, Token.tochars[kw].length, cast(uint)kw); 597 } 598 } 599 600 /************************************ 601 * This is used to pick the C keywords out of the tokens. 602 * If it's not a C keyword, then it's an identifier. 603 */ 604 static immutable TOK[TOK.max + 1] Ckeywords = 605 () { 606 with (TOK) 607 { 608 TOK[TOK.max + 1] tab = identifier; // default to identifier 609 enum Ckwds = [ auto_, break_, case_, char_, const_, continue_, default_, do_, float64, else_, 610 enum_, extern_, float32, for_, goto_, if_, inline, int32, int64, register, 611 restrict, return_, int16, signed, sizeof_, static_, struct_, switch_, typedef_, 612 union_, unsigned, void_, volatile, while_, asm_, typeof_, 613 _Alignas, _Alignof, _Atomic, _Bool, _Complex, _Generic, _Imaginary, _Noreturn, 614 _Static_assert, _Thread_local, 615 _import, __cdecl, __declspec, __stdcall, __thread, __pragma, __int128, __attribute__, 616 _assert ]; 617 618 foreach (kw; Ckwds) 619 tab[kw] = cast(TOK) kw; 620 621 return tab; 622 } 623 } (); 624 625 626 /*********************************************************** 627 */ 628 extern (C++) struct Token 629 { 630 Token* next; 631 Loc loc; 632 const(char)* ptr; // pointer to first character of this token within buffer 633 TOK value; 634 const(char)[] blockComment; // doc comment string prior to this token 635 const(char)[] lineComment; // doc comment for previous token 636 637 union 638 { 639 // Integers 640 long intvalue; 641 ulong unsvalue; 642 // Floats 643 real_t floatvalue; 644 645 struct 646 { 647 const(char)* ustring; // UTF8 string 648 uint len; 649 ubyte postfix; // 'c', 'w', 'd' 650 } 651 652 Identifier ident; 653 } 654 655 extern (D) private static immutable string[TOK.max + 1] tochars = 656 [ 657 // Keywords 658 TOK.this_: "this", 659 TOK.super_: "super", 660 TOK.assert_: "assert", 661 TOK.null_: "null", 662 TOK.true_: "true", 663 TOK.false_: "false", 664 TOK.cast_: "cast", 665 TOK.new_: "new", 666 TOK.delete_: "delete", 667 TOK.throw_: "throw", 668 TOK.module_: "module", 669 TOK.pragma_: "pragma", 670 TOK.typeof_: "typeof", 671 TOK.typeid_: "typeid", 672 TOK.template_: "template", 673 TOK.void_: "void", 674 TOK.int8: "byte", 675 TOK.uns8: "ubyte", 676 TOK.int16: "short", 677 TOK.uns16: "ushort", 678 TOK.int32: "int", 679 TOK.uns32: "uint", 680 TOK.int64: "long", 681 TOK.uns64: "ulong", 682 TOK.int128: "cent", 683 TOK.uns128: "ucent", 684 TOK.float32: "float", 685 TOK.float64: "double", 686 TOK.float80: "real", 687 TOK.bool_: "bool", 688 TOK.char_: "char", 689 TOK.wchar_: "wchar", 690 TOK.dchar_: "dchar", 691 TOK.imaginary32: "ifloat", 692 TOK.imaginary64: "idouble", 693 TOK.imaginary80: "ireal", 694 TOK.complex32: "cfloat", 695 TOK.complex64: "cdouble", 696 TOK.complex80: "creal", 697 TOK.delegate_: "delegate", 698 TOK.function_: "function", 699 TOK.is_: "is", 700 TOK.if_: "if", 701 TOK.else_: "else", 702 TOK.while_: "while", 703 TOK.for_: "for", 704 TOK.do_: "do", 705 TOK.switch_: "switch", 706 TOK.case_: "case", 707 TOK.default_: "default", 708 TOK.break_: "break", 709 TOK.continue_: "continue", 710 TOK.synchronized_: "synchronized", 711 TOK.return_: "return", 712 TOK.goto_: "goto", 713 TOK.try_: "try", 714 TOK.catch_: "catch", 715 TOK.finally_: "finally", 716 TOK.with_: "with", 717 TOK.asm_: "asm", 718 TOK.foreach_: "foreach", 719 TOK.foreach_reverse_: "foreach_reverse", 720 TOK.scope_: "scope", 721 TOK.struct_: "struct", 722 TOK.class_: "class", 723 TOK.interface_: "interface", 724 TOK.union_: "union", 725 TOK.enum_: "enum", 726 TOK.import_: "import", 727 TOK.mixin_: "mixin", 728 TOK.static_: "static", 729 TOK.final_: "final", 730 TOK.const_: "const", 731 TOK.alias_: "alias", 732 TOK.override_: "override", 733 TOK.abstract_: "abstract", 734 TOK.debug_: "debug", 735 TOK.deprecated_: "deprecated", 736 TOK.in_: "in", 737 TOK.out_: "out", 738 TOK.inout_: "inout", 739 TOK.lazy_: "lazy", 740 TOK.auto_: "auto", 741 TOK.align_: "align", 742 TOK.extern_: "extern", 743 TOK.private_: "private", 744 TOK.package_: "package", 745 TOK.protected_: "protected", 746 TOK.public_: "public", 747 TOK.export_: "export", 748 TOK.invariant_: "invariant", 749 TOK.unittest_: "unittest", 750 TOK.version_: "version", 751 TOK.argumentTypes: "__argTypes", 752 TOK.parameters: "__parameters", 753 TOK.ref_: "ref", 754 TOK.macro_: "macro", 755 TOK.pure_: "pure", 756 TOK.nothrow_: "nothrow", 757 TOK.gshared: "__gshared", 758 TOK.traits: "__traits", 759 TOK.vector: "__vector", 760 TOK.file: "__FILE__", 761 TOK.fileFullPath: "__FILE_FULL_PATH__", 762 TOK.line: "__LINE__", 763 TOK.moduleString: "__MODULE__", 764 TOK.functionString: "__FUNCTION__", 765 TOK.prettyFunction: "__PRETTY_FUNCTION__", 766 TOK.shared_: "shared", 767 TOK.immutable_: "immutable", 768 769 TOK.endOfFile: "End of File", 770 TOK.leftCurly: "{", 771 TOK.rightCurly: "}", 772 TOK.leftParenthesis: "(", 773 TOK.rightParenthesis: ")", 774 TOK.leftBracket: "[", 775 TOK.rightBracket: "]", 776 TOK.semicolon: ";", 777 TOK.colon: ":", 778 TOK.comma: ",", 779 TOK.dot: ".", 780 TOK.xor: "^", 781 TOK.xorAssign: "^=", 782 TOK.assign: "=", 783 TOK.lessThan: "<", 784 TOK.greaterThan: ">", 785 TOK.lessOrEqual: "<=", 786 TOK.greaterOrEqual: ">=", 787 TOK.equal: "==", 788 TOK.notEqual: "!=", 789 TOK.not: "!", 790 TOK.leftShift: "<<", 791 TOK.rightShift: ">>", 792 TOK.unsignedRightShift: ">>>", 793 TOK.add: "+", 794 TOK.min: "-", 795 TOK.mul: "*", 796 TOK.div: "/", 797 TOK.mod: "%", 798 TOK.slice: "..", 799 TOK.dotDotDot: "...", 800 TOK.and: "&", 801 TOK.andAnd: "&&", 802 TOK.or: "|", 803 TOK.orOr: "||", 804 TOK.tilde: "~", 805 TOK.dollar: "$", 806 TOK.plusPlus: "++", 807 TOK.minusMinus: "--", 808 TOK.question: "?", 809 TOK.variable: "var", 810 TOK.addAssign: "+=", 811 TOK.minAssign: "-=", 812 TOK.mulAssign: "*=", 813 TOK.divAssign: "/=", 814 TOK.modAssign: "%=", 815 TOK.leftShiftAssign: "<<=", 816 TOK.rightShiftAssign: ">>=", 817 TOK.unsignedRightShiftAssign: ">>>=", 818 TOK.andAssign: "&=", 819 TOK.orAssign: "|=", 820 TOK.concatenateAssign: "~=", 821 TOK.identity: "is", 822 TOK.notIdentity: "!is", 823 TOK.identifier: "identifier", 824 TOK.at: "@", 825 TOK.pow: "^^", 826 TOK.powAssign: "^^=", 827 TOK.goesTo: "=>", 828 TOK.pound: "#", 829 TOK.arrow: "->", 830 TOK.colonColon: "::", 831 832 // For debugging 833 TOK.error: "error", 834 TOK.string_: "string", 835 TOK.onScopeExit: "scope(exit)", 836 TOK.onScopeSuccess: "scope(success)", 837 TOK.onScopeFailure: "scope(failure)", 838 839 // Finish up 840 TOK.reserved: "reserved", 841 TOK.comment: "comment", 842 TOK.int32Literal: "int32v", 843 TOK.uns32Literal: "uns32v", 844 TOK.int64Literal: "int64v", 845 TOK.uns64Literal: "uns64v", 846 TOK.int128Literal: "int128v", 847 TOK.uns128Literal: "uns128v", 848 TOK.float32Literal: "float32v", 849 TOK.float64Literal: "float64v", 850 TOK.float80Literal: "float80v", 851 TOK.imaginary32Literal: "imaginary32v", 852 TOK.imaginary64Literal: "imaginary64v", 853 TOK.imaginary80Literal: "imaginary80v", 854 TOK.charLiteral: "charv", 855 TOK.wcharLiteral: "wcharv", 856 TOK.dcharLiteral: "dcharv", 857 TOK.wchar_tLiteral: "wchar_tv", 858 TOK.endOfLine: "\\n", 859 TOK.whitespace: "whitespace", 860 861 // C only keywords 862 TOK.inline : "inline", 863 TOK.register : "register", 864 TOK.restrict : "restrict", 865 TOK.signed : "signed", 866 TOK.sizeof_ : "sizeof", 867 TOK.typedef_ : "typedef", 868 TOK.unsigned : "unsigned", 869 TOK..volatile : "volatile", 870 TOK._Alignas : "_Alignas", 871 TOK._Alignof : "_Alignof", 872 TOK._Atomic : "_Atomic", 873 TOK._Bool : "_Bool", 874 TOK._Complex : "_Complex", 875 TOK._Generic : "_Generic", 876 TOK._Imaginary: "_Imaginary", 877 TOK._Noreturn : "_Noreturn", 878 TOK._Static_assert : "_Static_assert", 879 TOK._Thread_local : "_Thread_local", 880 881 // C only extended keywords 882 TOK._assert : "__check", 883 TOK._import : "__import", 884 TOK.__cdecl : "__cdecl", 885 TOK.__declspec : "__declspec", 886 TOK.__stdcall : "__stdcall", 887 TOK.__thread : "__thread", 888 TOK.__pragma : "__pragma", 889 TOK.__int128 : "__int128", 890 TOK.__attribute__ : "__attribute__", 891 ]; 892 893 static assert(() { 894 foreach (s; tochars) 895 assert(s.length); 896 return true; 897 }()); 898 899 nothrow: 900 901 int isKeyword() const 902 { 903 foreach (kw; keywords) 904 { 905 if (kw == value) 906 return 1; 907 } 908 return 0; 909 } 910 911 /**** 912 * Set to contents of ptr[0..length] 913 * Params: 914 * ptr = pointer to string 915 * length = length of string 916 */ 917 void setString(const(char)* ptr, size_t length) 918 { 919 auto s = cast(char*)mem.xmalloc_noscan(length + 1); 920 memcpy(s, ptr, length); 921 s[length] = 0; 922 ustring = s; 923 len = cast(uint)length; 924 postfix = 0; 925 } 926 927 /**** 928 * Set to contents of buf 929 * Params: 930 * buf = string (not zero terminated) 931 */ 932 void setString(const ref OutBuffer buf) 933 { 934 setString(cast(const(char)*)buf[].ptr, buf.length); 935 } 936 937 /**** 938 * Set to empty string 939 */ 940 void setString() 941 { 942 ustring = ""; 943 len = 0; 944 postfix = 0; 945 } 946 947 extern (C++) const(char)* toChars() const 948 { 949 const bufflen = 3 + 3 * floatvalue.sizeof + 1; 950 __gshared char[bufflen] buffer; 951 const(char)* p = &buffer[0]; 952 switch (value) 953 { 954 case TOK.int32Literal: 955 snprintf(&buffer[0], bufflen, "%d", cast(int)intvalue); 956 break; 957 case TOK.uns32Literal: 958 case TOK.wchar_tLiteral: 959 snprintf(&buffer[0], bufflen, "%uU", cast(uint)unsvalue); 960 break; 961 case TOK.wcharLiteral: 962 case TOK.dcharLiteral: 963 case TOK.charLiteral: 964 { 965 OutBuffer buf; 966 buf.writeSingleCharLiteral(cast(dchar) intvalue); 967 buf.writeByte('\0'); 968 p = buf.extractChars(); 969 } 970 break; 971 case TOK.int64Literal: 972 snprintf(&buffer[0], bufflen, "%lldL", cast(long)intvalue); 973 break; 974 case TOK.uns64Literal: 975 snprintf(&buffer[0], bufflen, "%lluUL", cast(ulong)unsvalue); 976 break; 977 case TOK.float32Literal: 978 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 979 strcat(&buffer[0], "f"); 980 break; 981 case TOK.float64Literal: 982 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 983 break; 984 case TOK.float80Literal: 985 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 986 strcat(&buffer[0], "L"); 987 break; 988 case TOK.imaginary32Literal: 989 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 990 strcat(&buffer[0], "fi"); 991 break; 992 case TOK.imaginary64Literal: 993 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 994 strcat(&buffer[0], "i"); 995 break; 996 case TOK.imaginary80Literal: 997 CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue); 998 strcat(&buffer[0], "Li"); 999 break; 1000 case TOK.string_: 1001 { 1002 OutBuffer buf; 1003 buf.writeByte('"'); 1004 for (size_t i = 0; i < len;) 1005 { 1006 dchar c; 1007 utf_decodeChar(ustring[0 .. len], i, c); 1008 writeCharLiteral(buf, c); 1009 } 1010 buf.writeByte('"'); 1011 if (postfix) 1012 buf.writeByte(postfix); 1013 buf.writeByte(0); 1014 p = buf.extractChars(); 1015 } 1016 break; 1017 case TOK.identifier: 1018 case TOK.enum_: 1019 case TOK.struct_: 1020 case TOK.import_: 1021 case TOK.wchar_: 1022 case TOK.dchar_: 1023 case TOK.bool_: 1024 case TOK.char_: 1025 case TOK.int8: 1026 case TOK.uns8: 1027 case TOK.int16: 1028 case TOK.uns16: 1029 case TOK.int32: 1030 case TOK.uns32: 1031 case TOK.int64: 1032 case TOK.uns64: 1033 case TOK.int128: 1034 case TOK.uns128: 1035 case TOK.float32: 1036 case TOK.float64: 1037 case TOK.float80: 1038 case TOK.imaginary32: 1039 case TOK.imaginary64: 1040 case TOK.imaginary80: 1041 case TOK.complex32: 1042 case TOK.complex64: 1043 case TOK.complex80: 1044 case TOK.void_: 1045 p = ident.toChars(); 1046 break; 1047 default: 1048 p = toChars(value); 1049 break; 1050 } 1051 return p; 1052 } 1053 1054 static const(char)* toChars(TOK value) 1055 { 1056 return toString(value).ptr; 1057 } 1058 1059 extern (D) static string toString(TOK value) pure nothrow @nogc @safe 1060 { 1061 return tochars[value]; 1062 } 1063 } 1064 1065 /** 1066 * Write a character, using a readable escape sequence if needed 1067 * 1068 * Useful for printing "" string literals in e.g. error messages, ddoc, or the `.stringof` property 1069 * 1070 * Params: 1071 * buf = buffer to append character in 1072 * c = code point to write 1073 */ 1074 nothrow 1075 void writeCharLiteral(ref OutBuffer buf, dchar c) 1076 { 1077 switch (c) 1078 { 1079 case '\0': 1080 buf.writestring("\\0"); 1081 break; 1082 case '\n': 1083 buf.writestring("\\n"); 1084 break; 1085 case '\r': 1086 buf.writestring("\\r"); 1087 break; 1088 case '\t': 1089 buf.writestring("\\t"); 1090 break; 1091 case '\b': 1092 buf.writestring("\\b"); 1093 break; 1094 case '\f': 1095 buf.writestring("\\f"); 1096 break; 1097 case '"': 1098 case '\\': 1099 buf.writeByte('\\'); 1100 goto default; 1101 default: 1102 if (c <= 0xFF) 1103 { 1104 if (isprint(c)) 1105 buf.writeByte(c); 1106 else 1107 buf.printf("\\x%02x", c); 1108 } 1109 else if (c <= 0xFFFF) 1110 buf.printf("\\u%04x", c); 1111 else 1112 buf.printf("\\U%08x", c); 1113 break; 1114 } 1115 } 1116 1117 unittest 1118 { 1119 OutBuffer buf; 1120 foreach(dchar d; "a\n\r\t\b\f\0\x11\u7233\U00017233"d) 1121 { 1122 writeCharLiteral(buf, d); 1123 } 1124 assert(buf[] == `a\n\r\t\b\f\0\x11\u7233\U00017233`); 1125 } 1126 1127 /** 1128 * Write a single-quoted character literal 1129 * 1130 * Useful for printing '' char literals in e.g. error messages, ddoc, or the `.stringof` property 1131 * 1132 * Params: 1133 * buf = buffer to append character in 1134 * c = code point to write 1135 */ 1136 nothrow 1137 void writeSingleCharLiteral(ref OutBuffer buf, dchar c) 1138 { 1139 buf.writeByte('\''); 1140 if (c == '\'') 1141 buf.writeByte('\\'); 1142 1143 if (c == '"') 1144 buf.writeByte('"'); 1145 else 1146 writeCharLiteral(buf, c); 1147 1148 buf.writeByte('\''); 1149 } 1150 1151 unittest 1152 { 1153 OutBuffer buf; 1154 writeSingleCharLiteral(buf, '\''); 1155 assert(buf[] == `'\''`); 1156 buf.reset(); 1157 writeSingleCharLiteral(buf, '"'); 1158 assert(buf[] == `'"'`); 1159 buf.reset(); 1160 writeSingleCharLiteral(buf, '\n'); 1161 assert(buf[] == `'\n'`); 1162 }