x64 NASM汇编程序执行异常：合法十六进制输入被判定无效

问题描述 #

编写了一份兼容NASM的x64 Linux汇编代码，功能是接收十六进制字符串输入，验证合法性后与0x100比较，相等则输出成功消息，否则输出失败消息。但无论输入合法还是非法十六进制值，程序都会输出invalid hex code。

原始代码 #

global _start

section .data

    hex_chars db "ABCDEF0123456789"         ; valid hex characters

    success_message: db "Secret Unlocked", 0x0a     ; success message
    fail_message: db "Vault Locked", 0x0a           ; failed message

    success_message_length: equ $ - success_message
    fail_message_length: equ $ - fail_message

    error_message: db "invalid hex code", 0x0a
    error_message_size: equ $ - error_message

section .bss

    input_buffer resq 16

section .text

_start:

    ; --- main code here --- ;

    ; getting the user input: syscall-read

    mov rax, 0x00
    mov rdi, 0x00
    mov rsi, input_buffer
    mov rdx, 16
    syscall

    mov rdi, rsi            ; hex_to_int takes an value from rdi
    call hex_to_int         ; calling the hex to integer

    ; hex value is in the rax

    cmp rax, 0x100          ; compare the value in rax with 0x100 hex
    je success          ; jump to success function (exit from there)

    ; program will reach here if number is not 0x100, so throw an failed message

    mov rax, 0x01
    mov rdi, 0x01
    mov rsi, fail_message
    mov rdx, fail_message_length
    syscall

    ; exiting with code 0

    mov rax, 0x3c
    mov rdi, 0x00
    syscall

success:

    ; success message

    mov rax, 0x01
    mov rdi, 0x01
    mov rsi, success_message
    mov rdx, success_message_length
    syscall

    ; exiting with code 0

    mov rax, 0x3c
    mov rdi, 0x00
    syscall

hex_to_int:

    ; --- init values --- ;

    xor rax, rax            ; clearing the rax (accumulator)
    xor rcx, rcx            ; clearing the rcx (loop counter)

convert_loop:

    movzx rdi, byte [rdi + rcx] ; get the char from the string
    test rdi, rdi           ; check if it is a null ptr (end of the string)
    jz done             ; if yes, jump to done

    ; check if the char is a valid hex value

    mov rbx, hex_chars      ; load the test values to rbx
    mov rsi, 0          ; init rsi = 0

; rdi contains the char
; rbx contains the hex_chars
; rcx is the program counter

search_loop:

    cmp rdi, [rbx + rsi]        ; take a single char from the hex_value and compare to the char
    je digit_found          ; jump to digit_found function if found

    ; not found!

    add rsi, 1          ; getting ready for the next test char
    cmp rsi, 16         ; check if all hex test cases are completed
    jne search_loop         ; jump to check next hex test char if rsi != 16

    ; if program reaches this point, then exit with an error (not test cases passed -> invalid hex char)

    ; printing the invalid message

    mov rax, 0x01
    mov rdi, 0x01
    mov rsi, error_message
    mov rdx, error_message_size
    syscall

    ; exit with exit code 0

    mov rax, 0x3c
    mov rdi, 0x00
    syscall

digit_found:

    imul rdi, rdi, 16       ; rax = rax * 16
    add rax, rdi            ; building up the hex value

    inc rcx             ; incrementing the program counter
    jmp convert_loop        ; loop back the the convert loop section

done:

    ret

; ---------------------------------------- Notes ---------------------------------------- ;
;
; when refering to a pointer holding some value, it's actually the memory address the registers
; are holding with it.
; Example:
;   rbx = "ABCEDF"
;   rbx is pointing to A
;   rbx + 1 = B
;
;   hence loop over: rsi = 0 to rsi = 6
;       [ rbx + rsi ] will fetch desired values
;
; --------------------------------------------------------------------------------------- ;

核心错误点 #

• 寄存器被意外覆盖：在convert_loop中，movzx rdi, byte [rdi + rcx]把输入字符串的指针（原本存在rdi里）替换成了当前字符的ASCII值，后续循环时[rdi + rcx]不再指向输入缓冲区，导致读取错误内存。
• 操作数大小不匹配：cmp rdi, [rbx + rsi]默认比较64位数据，但我们只需要比较单字节的字符，应该用cmp dil, byte [rbx + rsi]（dil是rdi的低8位）。
• 数值转换逻辑错误：digit_found里的imul rdi, rdi, 16完全错误，注释写的是rax = rax *16但实际修改的是rdi；而且直接将字符的ASCII码加入rax，没有转换成对应的十六进制数值（比如'A'的ASCII是0x41，对应十六进制值10）。
• 未处理换行符：read系统调用会把用户输入的换行符（0x0a）也读入缓冲区，这个字符不在hex_chars里，导致误判为非法字符。

修复后的代码 #

global _start

section .data

    hex_chars db "ABCDEFabcdef0123456789"  ; 支持大小写十六进制字符
    hex_count equ $ - hex_chars            ; 合法字符数量

    success_message: db "Secret Unlocked", 0x0a
    success_message_length: equ $ - success_message

    fail_message: db "Vault Locked", 0x0a
    fail_message_length: equ $ - fail_message

    error_message: db "invalid hex code", 0x0a
    error_message_size: equ $ - error_message

section .bss

    input_buffer resb 16    ; 用resb更合理，因为是字节缓冲区

section .text

_start:
    ; 读取用户输入
    mov rax, 0x00           ; sys_read
    mov rdi, 0x00           ; stdin
    mov rsi, input_buffer
    mov rdx, 16
    syscall

    ; rax是读取的字节数，存储到r8备用
    mov r8, rax

    ; 调用hex_to_int，rdi传输入缓冲区指针
    mov rdi, input_buffer
    call hex_to_int

    ; 比较转换后的值与0x100
    cmp rax, 0x100
    je success

    ; 输出失败消息
    mov rax, 0x01           ; sys_write
    mov rdi, 0x01           ; stdout
    mov rsi, fail_message
    mov rdx, fail_message_length
    syscall

    ; 退出
    mov rax, 0x3c
    mov rdi, 0x00
    syscall

success:
    ; 输出成功消息
    mov rax, 0x01
    mov rdi, 0x01
    mov rsi, success_message
    mov rdx, success_message_length
    syscall

    mov rax, 0x3c
    mov rdi, 0x00
    syscall

hex_to_int:
    xor rax, rax            ; rax存储转换后的数值
    xor rcx, rcx            ; rcx是循环计数器

convert_loop:
    ; 检查是否已经处理完所有读取的字符
    cmp rcx, r8
    jge done

    ; 读取当前字符到dil（rdi的低8位），保留rdi作为缓冲区指针
    mov dil, byte [rdi + rcx]

    ; 处理换行符，视为输入结束
    cmp dil, 0x0a
    je done

    ; 检查是否是合法十六进制字符
    mov rbx, hex_chars
    mov rsi, 0

search_loop:
    cmp rsi, hex_count
    jge invalid_char        ; 遍历完所有合法字符仍未找到，判定非法

    cmp dil, byte [rbx + rsi]
    je digit_found

    inc rsi
    jmp search_loop

digit_found:
    ; 将rsi转换为对应的十六进制数值：0-9对应0-9，A-F/a-f对应10-15
    mov r9, rsi
    cmp rsi, 10
    jl digit_value          ; 0-9直接用rsi的值
    sub r9, 10              ; A-F/a-f减去10得到10-15
    add r9, 10

digit_value:
    ; rax = rax * 16 + 当前数值
    imul rax, rax, 16
    add rax, r9

    inc rcx
    jmp convert_loop

invalid_char:
    ; 输出错误消息并退出
    mov rax, 0x01
    mov rdi, 0x01
    mov rsi, error_message
    mov rdx, error_message_size
    syscall

    mov rax, 0x3c
    mov rdi, 0x00
    syscall

done:
    ret

修复说明 #

• 保留rdi作为输入缓冲区指针，用dil存储当前字符，避免指针被覆盖。
• 修正操作数大小，用cmp dil, byte [rbx + rsi]进行单字节比较。
• 正确实现十六进制数值转换：根据字符在hex_chars中的索引计算对应的数值，0-9直接取索引，A-F/a-f取索引减10加10（即10-15）。
• 处理read返回的字节数，以及输入末尾的换行符，遇到换行符视为输入结束。
• 扩展hex_chars支持大小写十六进制字符，提升兼容性。

内容的提问来源于stack exchange，提问作者Aditya Patil