_Core/Lang/Parser/Lexer.cs

这里只显示每个文件的开头 60 行。登录后可以解锁完整代码。

微信扫码关注公众号，代码整理好后第一时间通知你。

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;


namespace ComLib.Lang
{
    /// <summary>
    /// Converts script from a series of characters into a series of tokens.
    /// Main method is NextToken();
    /// A script can be broken down into a sequence of tokens.
    /// e.g.
    /// 
    /// 1. var name = "kishore";
    /// Tokens:
    /// 
    ///  TOKEN VALUE:         TOKEN TYPE:
    ///  var         keyword
    ///  ""          literal ( whitespace )
    ///  name        id
    ///  ""          literal ( whitespace )
    ///  =           operator
    ///  ""          literal ( whitespace )
    ///  "kishore"   literal
    ///  ;           operator
    /// </summary>
    public class Lexer : LexerBase
    {
        enum TokenLengthCalcMode
        {
            /// <summary>
            /// Use the size the token text
            /// </summary>
            Direct,


            /// <summary>
            /// Use the length of whitespace
            /// </summary>
            WhiteSpace,


            /// <summary>
            /// Use the length of the token plus 2 chars for string literal
            /// </summary>
            String,


            /// <summary>
            /// MUltiline comment.
            /// </summary>
            MultilineComment
        }


        #region Private members
        private Context _ctx;      
        private Token _lastToken;
        private TokenData _lastTokenData;