如何在C#控制台文本编辑器中实现文本高亮？寻求AST及替代方案

1. 构建AST的步骤 #

要利用AST实现精确的语法高亮，你需要在现有分词器基础上添加解析器，将线性Token序列转换为带层级结构的抽象语法树：

步骤1：扩展Token类型，补充上下文信息

你的现有分词器仅区分基础Token类型（如Open、Close），缺少标签名、属性名等关键上下文。首先扩展HTMLTokenType：

public enum HTMLTokenType
{
    End,
    Quotes,
    Equal,
    AttributeName, // 新增：属性名
    AttributeValue, // 新增：属性值
    TagName, // 新增：标签名
    CloseTag, // 新增：闭合标签标记
    OpenTag, // 新增：开始标签标记
    TextContent, // 新增：文本内容
}

同时修改分词逻辑，将原本归为String的内容进一步区分为标签名、属性名、属性值和普通文本。

步骤2：定义AST节点类

创建对应HTML结构的AST节点，体现层级关系：

public abstract class AstNode
{
    public int Start { get; set; }
    public int Length { get; set; }
}

public class ElementNode : AstNode
{
    public string TagName { get; set; }
    public List<AttributeNode> Attributes { get; set; } = new();
    public List<AstNode> Children { get; set; } = new();
}

public class AttributeNode : AstNode
{
    public string Name { get; set; }
    public string Value { get; set; }
}

public class TextNode : AstNode
{
    public string Content { get; set; }
}

步骤3：实现解析器构建AST

解析器通过栈结构跟踪标签嵌套关系，消费Token并构建AST：

public class HTMLParser
{
    private readonly List<Token> _tokens;
    private readonly string _originalText;
    private int _currentIndex;

    public HTMLParser(List<Token> tokens, string originalText)
    {
        _tokens = tokens;
        _originalText = originalText;
    }

    public ElementNode Parse()
    {
        var root = new ElementNode { TagName = "root" };
        var nodeStack = new Stack<ElementNode>();
        nodeStack.Push(root);

        while (_currentIndex < _tokens.Count)
        {
            var token = _tokens[_currentIndex];
            switch (token.Type)
            {
                case HTMLTokenType.OpenTag:
                    _currentIndex++;
                    var tagToken = _tokens[_currentIndex];
                    var element = new ElementNode
                    {
                        TagName = tagToken.Type == HTMLTokenType.TagName 
                            ? _originalText.Substring(tagToken.Start, tagToken.Length) : "",
                        Start = token.Start,
                        Length = tagToken.Start + tagToken.Length - token.Start
                    };
                    nodeStack.Peek().Children.Add(element);
                    nodeStack.Push(element);
                    _currentIndex++;
                    break;
                case HTMLTokenType.CloseTag:
                    _currentIndex++;
                    nodeStack.Pop();
                    _currentIndex++; // 跳过闭合标签的TagName
                    break;
                case HTMLTokenType.AttributeName:
                    var attrName = _originalText.Substring(token.Start, token.Length);
                    _currentIndex++;
                    _currentIndex++; // 跳过Equal符号
                    _currentIndex++; // 跳过Quotes
                    var attrValueToken = _tokens[_currentIndex];
                    var attrValue = _originalText.Substring(attrValueToken.Start, attrValueToken.Length);
                    nodeStack.Peek().Attributes.Add(new AttributeNode
                    {
                        Name = attrName,
                        Value = attrValue,
                        Start = token.Start,
                        Length = attrValueToken.Start + attrValueToken.Length - token.Start
                    });
                    _currentIndex += 2; // 跳过Value和闭合Quotes
                    break;
                case HTMLTokenType.TextContent:
                    nodeStack.Peek().Children.Add(new TextNode
                    {
                        Content = _originalText.Substring(token.Start, token.Length),
                        Start = token.Start,
                        Length = token.Length
                    });
                    _currentIndex++;
                    break;
                default:
                    _currentIndex++;
                    break;
            }
        }
        return root;
    }
}

步骤4：遍历AST应用高亮

构建完AST后，遍历每个节点，根据节点类型和上下文分配高亮样式：

• 标签名：紫色加粗
• 属性名：蓝绿色
• 属性值：蓝色
• 文本内容：默认黑色

如果不想构建完整AST，可以采用状态机驱动的上下文感知分词，通过维护当前状态和标签栈，为Token添加语义标记：

核心思路 #

1. 维护状态栈，记录当前语法上下文（如InOpenTag、InAttributeValue）
2. 维护标签栈，跟踪当前打开的标签列表
3. 分词时根据状态和位置，为Token添加具体语义（如TagName_H1、AttributeName_Class）

示例实现（状态机逻辑） #

public enum LexerState
{
    Default,
    InOpenTag,
    InCloseTag,
    InAttributeName,
    InAttributeValue
}

public class ContextualHTMLLexer
{
    private readonly string _text;
    private int _cursor;
    private LexerState _currentState = LexerState.Default;
    private Stack<string> _tagStack = new();

    private readonly Dictionary<string, (HTMLTokenType, int)> _keywords = new()
    {
        { "<", (HTMLTokenType.OpenTag, 1) },
        { "</", (HTMLTokenType.CloseTag, 2) },
        { "/>", (HTMLTokenType.Close, 2) },
        { ">", (HTMLTokenType.Close, 1) },
        { "=", (HTMLTokenType.Equal, 1) },
        { "\"", (HTMLTokenType.Quotes, 1) },
    };

    public ContextualHTMLLexer(string text)
    {
        _text = text;
    }

    public Token NextToken()
    {
        if (_cursor >= _text.Length)
            return new Token(HTMLTokenType.End, _cursor, 0);

        // 匹配双字符关键词
        if (_cursor + 1 < _text.Length && _keywords.ContainsKey(_text.Substring(_cursor, 2)))
        {
            var (type, length) = _keywords[_text.Substring(_cursor, 2)];
            UpdateState(type);
            var token = new Token(type, _cursor, length);
            _cursor += length;
            return token;
        }

        // 匹配单字符关键词
        if (_keywords.ContainsKey(_text[_cursor].ToString()))
        {
            var (type, length) = _keywords[_text[_cursor].ToString()];
            UpdateState(type);
            var token = new Token(type, _cursor, length);
            _cursor += length;
            return token;
        }

        // 处理文本类Token
        int start = _cursor;
        while (_cursor < _text.Length)
        {
            if (_text[_cursor] == ' ' || 
                (_keywords.ContainsKey(_text[_cursor].ToString()) || 
                (_cursor +1 < _text.Length && _keywords.ContainsKey(_text.Substring(_cursor,2)))))
                break;
            _cursor++;
        }
        var tokenType = GetTextTokenType();
        return new Token(tokenType, start, _cursor - start);
    }

    private void UpdateState(HTMLTokenType tokenType)
    {
        switch (_currentState)
        {
            case LexerState.Default:
                if (tokenType == HTMLTokenType.OpenTag)
                    _currentState = LexerState.InOpenTag;
                break;
            case LexerState.InOpenTag:
                if (tokenType == HTMLTokenType.Close)
                    _currentState = LexerState.Default;
                else if (tokenType == HTMLTokenType.Equal)
                    _currentState = LexerState.InAttributeValue;
                break;
            case LexerState.InAttributeValue:
                if (tokenType == HTMLTokenType.Quotes)
                    _currentState = _currentState == LexerState.InAttributeValue 
                        ? LexerState.InOpenTag : LexerState.InAttributeValue;
                break;
        }
    }

    private HTMLTokenType GetTextTokenType()
    {
        if (_currentState == LexerState.InOpenTag || _currentState == LexerState.InCloseTag)
            return HTMLTokenType.TagName;
        else if (_currentState == LexerState.InAttributeName)
            return HTMLTokenType.AttributeName;
        else if (_currentState == LexerState.InAttributeValue)
            return HTMLTokenType.AttributeValue;
        return HTMLTokenType.TextContent;
    }
}

以下是将Token转换为高亮HTML的前端实现：

HTML结构 #

<div class="editor">
    <pre id="highlighted-code"></pre>
</div>

CSS样式 #

.editor {
    font-family: 'Consolas', monospace;
    font-size: 14px;
    padding: 16px;
    background: #f8f8f8;
    border-radius: 4px;
}

.tag-name { color: #800080; font-weight: bold; }
.attribute-name { color: #008080; }
.attribute-value { color: #0000ff; }
.text-content { color: #333333; }
.tag-delimiter { color: #808080; }

JavaScript逻辑 #

function renderHighlight(tokens, originalText) {
    const container = document.getElementById('highlighted-code');
    let html = '';

    tokens.forEach(token => {
        const text = originalText.slice(token.start, token.start + token.length);
        let cls = '';

        switch(token.type) {
            case 'TagName': cls = 'tag-name'; break;
            case 'AttributeName': cls = 'attribute-name'; break;
            case 'AttributeValue': cls = 'attribute-value'; break;
            case 'TextContent': cls = 'text-content'; break;
            case 'OpenTag': case 'CloseTag': case 'Quotes': case 'Equal':
                cls = 'tag-delimiter'; break;
        }

        html += `<span class="${cls}">${escapeHtml(text)}</span>`;
    });

    container.innerHTML = html;
}

function escapeHtml(str) {
    return str.replace(/&/g, '&amp;')
              .replace(/</g, '&lt;')
              .replace(/>/g, '&gt;')
              .replace(/"/g, '&quot;');
}

内容的提问来源于stack exchange，提问作者Rostislav Kiv