dlsym crash when called from assembler

I have a small program in assembler that loads an .so file using dlopen, and then tries to load a function pointer using dlsym. Calling dlopen seems to be fine but it crashes when I call dlsym.


;default rel

EXTERN dlopen ; loads a dynamic library
EXTERN dlsym ; retrieves the address for a symbol in the dynamic library

; inputs:
;   rdi: rdi the pointer to print
    sub rsp, 19 ; allocate space for the string 0x0123456789ABCDEF\n
    mov BYTE [rsp + 0], '0'
    mov BYTE [rsp + 1], 'x'
    xor rcx, rcx ; int loop variable to 0
        lea rsi, [rsp + rcx] ; rsi will we the offset where we will store the next hex charcter
        mov rax, rdi
        and rax, 0xf
        sar rdi, 4 ; shift right 4 bits (divide by 16)
        lea rdx, [hexLookUp + rax]
        mov bl, [rdx]
        mov BYTE [rsi +18], bl
        dec rcx ; rcx--
        cmp rcx, -16 ; while rcx > -16
        jne .LOOP1
    mov BYTE [rsp + 18], 10

    ; print
    mov rax, 1 ; syscall: write
    mov rdi, 1 ; stdout
    mov rsi, rsp
    mov rdx, 19
    ; release stack memory
    add rsp, 19

global _start ; "global" means that the symbol can be accessed in other modules. In order to refer to a global symbol from another module, you must use the "extern" keyboard

    ; load the library
    mov rdi, str_libX11so
    mov rsi, 2; RTLD_NOW=2
    call dlopen wrt ..plt
        ; PLT stands for Procedure Linkage Table:
        ; used to call external library functions whose address is not know at link time,
        ; so it must be resolved by the dynamic linker at run time
        ; more info: https://reverseengineering.stackexchange.com/questions/1992/what-is-plt-got
    mov [ptr_libX11so], rax ; the previous function call returned the value in rax
    mov rdi, rax 
    call printHex

    ; load the function
    mov rdi, [str_libX11so]
    mov rsi, fstr_XOpenDisplay
    call dlsym wrt ..plt
    mov [fptr_XOpenDisplay], rax
    mov rdi, rax
    call printHex

    mov rax, 60 ; syscal: exit
    mov rdi, 0 ; return code

hexLookUp: db "0123456789ABCDEF"
str_libX11so: db "libX11.so", 0

; X11 function names
fstr_XOpenDisplay: db "XOpenDisplay", 0

ptr_libX11so: dq 0 ; ptr to the X11 library

; X11 function ptrs
fptr_XOpenDisplay: dq 0

I have tried to make the same program in C and it seems to work. So I must be doing something wrong.

extern void* dlopen(const char* name, int);
extern void* dlsym(void* restrict handle, const char* restrict name);

int main()
    void* libX11so = dlopen("libX11.so", 2);
    void (*XOpenDisplay)() = dlsym(libX11so, "XOpenDisplay");

I tried to disassemble the C version and compare, but I can't still figure out what is the problem.

An interesting thing I noticed is that the pointer returned by dlopen (which is different in each execution), in the asm version is quite small compared to the C version (e.g 0x0000000001A932D vs 0x5555555592d0). But maybe that could be because I'm using the -no-pie flag for linking:

nasm -f elf64 -g -F dwarf minimal.asm && gcc -nostartfiles -no-pie minimal.o -ldl -o minimal && ./minimal

1 answer

  • answered 2021-01-19 21:52 tuket

    I just noticed my mistake:

        ; load the function
        mov rdi, [str_libX11so]

    should be:

        ; load the function
        mov rdi, [ptr_libX11so]