Lexer

nothrow

class Lexer {

Loc scanloc;

Loc prevloc;

const(char)* p;

Token token;

bool Ccompile;

ubyte boolsize;

ubyte shortsize;

ubyte intsize;

ubyte longsize;

ubyte long_longsize;

ubyte long_doublesize;

ubyte wchar_tsize;

ErrorSink eSink;

CompileEnv compileEnv;

this(const(char)* filename, const(char)* base, size_t begoffset, size_t endoffset, bool doDocComment, bool commentToken, ErrorSink errorSink, CompileEnv* compileEnv);

this(const(char)* filename, const(char)* base, size_t begoffset, size_t endoffset, bool doDocComment, bool commentToken, bool whitespaceToken, ErrorSink errorSink, CompileEnv* compileEnv);

this(ErrorSink errorSink);

void resetDefineLines(const(char)[] slice);

void nextDefineLine();

bool empty();

Token* allocateToken();

TOK peekNext();

TOK peekNext2();

void scan(Token* t);

Token* peekPastParen(Token* tk);

bool parseSpecialTokenSequence();

void poundLine(Token tok, bool linemarker);

void skipToNextLine(OutBuffer* defines);

static const(char)* combineComments(const(char)[] c1, const(char)[] c2, bool newParagraph);

}

Constructors

this this(const(char)* filename, const(char)* base, size_t begoffset, size_t endoffset, bool doDocComment, bool commentToken, ErrorSink errorSink, CompileEnv* compileEnv): Creates a Lexer for the source code base[begoffset..endoffset+1]. The last character, baseendoffset, must be null (0) or EOF (0x1A).
this this(const(char)* filename, const(char)* base, size_t begoffset, size_t endoffset, bool doDocComment, bool commentToken, bool whitespaceToken, ErrorSink errorSink, CompileEnv* compileEnv): Alternative entry point for DMDLIB, adds whitespaceToken
this this(ErrorSink errorSink): Used for unittests for a mock Lexer

Members

Functions

allocateToken Token* allocateToken()
empty bool empty(): Range interface
nextDefineLine void nextDefineLine(): Set up for next #define line. p should be at start of next line.
parseSpecialTokenSequence bool parseSpecialTokenSequence(): Parse special token sequence:
peekNext TOK peekNext(): Look ahead at next token's value.
peekNext2 TOK peekNext2(): Look 2 tokens ahead at value.
peekPastParen Token* peekPastParen(Token* tk): tk is on the opening (. Look ahead and return token that is past the closing ).
poundLine void poundLine(Token tok, bool linemarker): Parse line/file preprocessor directive: #line linnum filespec Allow __LINE__ for linnum, and __FILE__ for filespec. Accept linemarker format: # linnum filespec {flags} There can be zero or more flags, which are one of the digits 1..4, and must be in ascending order. The flags are ignored.
resetDefineLines void resetDefineLines(const(char)[] slice): Reset lexer to lex #define's
scan void scan(Token* t): Turn next token in buffer into a token.
skipToNextLine void skipToNextLine(OutBuffer* defines): Scan forward to start of next line.

Static functions

combineComments const(char)* combineComments(const(char)[] c1, const(char)[] c2, bool newParagraph): Combine two document comments into one, separated by an extra newline if newParagraph is true.

Variables

Ccompile bool Ccompile;: true if compiling ImportC
boolsize ubyte boolsize;: size of a C _Bool, default 1
compileEnv CompileEnv compileEnv;: environment
eSink ErrorSink eSink;: send error messages through this interface
intsize ubyte intsize;: size of a C int, default 4
long_doublesize ubyte long_doublesize;: size of C long double, 8 or D real.sizeof
long_longsize ubyte long_longsize;: size of a C long long, default 8
longsize ubyte longsize;: size of C long, 4 or 8
shortsize ubyte shortsize;: size of a C short, default 2
wchar_tsize ubyte wchar_tsize;: size of C wchar_t, 2 or 4

Meta

Source

See Implementation

dmd lexer classes

Lexer

structs

CompileEnv