DEV Community: ddupard

X86_64 From Source Code to Binary: What Does GCC Actually Do? part 3

ddupard — Thu, 23 Jul 2026 20:23:00 +0000

Having explained how GCC uses the System V AMD64 ABI convention for passing parameters , how GCC uses the stack to manage local variables and how GCC returns values from a function, it's time to explain some of the options GCC has to prevent the buffer overflow attack

To illustrate the buffer overflow protection mechanism, I will use the following program:


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <arpa/inet.h>

#define PORT 8080
#define BUFFER_SIZE 64


void test1() {
    char *t1 = "test1" ; 
    printf("%s\n",t1) ;
} 

void test2() {
    char *buffer1 = "test2\n" ;
    char *buffer2[10]  ;

    strcpy(buffer2[0],buffer1) ;
    printf("%s",buffer2[0]) ;
}



/**
 * handle_client: Processes incoming connections.
 * This function contains a deliberate stack-based buffer overflow 
 * vulnerability for educational purposes.
 */


void handle_client(int client_socket) {
    char buffer[BUFFER_SIZE];
    char request[256];

    // Read the incoming request from the client
    int bytes_received = recv(client_socket, request, sizeof(request) - 1, 0);
    if (bytes_received > 0) {
        request[bytes_received] = '\0';

        /* * THE VULNERABILITY: 
         * strcpy() does not perform bounds checking. If 'request' 
         * exceeds 64 bytes, it will overflow 'buffer' and overwrite 
         * critical data on the stack, including the saved return address.
         */
        strcpy(buffer, request);
        printf("Received: %s\n", buffer);

        test1() ;
        test2() ;



    }
    close(client_socket);
}

int main() {
    int server_fd = socket(AF_INET, SOCK_STREAM, 0);
    struct sockaddr_in address = {AF_INET, htons(PORT), INADDR_ANY};

    // Bind the socket to the port and start listening
    bind(server_fd, (struct sockaddr *)&address, sizeof(address));
    listen(server_fd, 3);

    printf("Server listening on port %d...\n", PORT);

    while (1) {
        int client_socket = accept(server_fd, NULL, NULL);
        handle_client(client_socket);
    }
    return 0;
}

This program will be compiled with GCC and the binary generated will be disassembled by objdump using the following commands

if [ ! -f  "$OUT/gcc_3/server.objdump_clean" ]; then
    gcc -g -fno-stack-protector -z execstack -no-pie  -o "$OUT/gcc_3/server" "$SRC/buffer_overflow/C/server.c"
    objdump -M intel --show-raw-insn -d "$OUT/gcc_3/server" > "$OUT/gcc_3/server.objdump"
    python3 "$ATP/clean_objdump.py" "$OUT/gcc_3/server.objdump" > "$OUT/gcc_3/server.objdump_clean"
fi

if [ ! -f  "$OUT/gcc_3/server_sp.objdump_clean" ]; then
    gcc -g -fstack-protector -z execstack -no-pie  -o "$OUT/gcc_3/server_sp" "$SRC/buffer_overflow/C/server.c"
    objdump -M intel --show-raw-insn -d "$OUT/gcc_3/server_sp" > "$OUT/gcc_3/server_sp.objdump"
    python3 "$ATP/clean_objdump.py" "$OUT/gcc_3/server_sp.objdump" > "$OUT/gcc_3/server_sp.objdump_clean"
fi

if [ ! -f  "$OUT/gcc_3/server_sps.objdump_clean" ]; then
    gcc -g -fstack-protector-strong -z execstack -no-pie  -o "$OUT/gcc_3/server_sps" "$SRC/buffer_overflow/C/server.c"
    objdump -M intel --show-raw-insn -d "$OUT/gcc_3/server_sps" > "$OUT/gcc_3/server_sps.objdump"
    python3 "$ATP/clean_objdump.py" "$OUT/gcc_3/server_sps.objdump" > "$OUT/gcc_3/server_sps.objdump_clean"
fi

if [ ! -f  "$OUT/gcc_3/server_spa.objdump_clean" ]; then
    gcc -g -fstack-protector-all -z execstack -no-pie  -o "$OUT/gcc_3/server_spa" "$SRC/buffer_overflow/C/server.c"
    objdump -M intel --show-raw-insn -d "$OUT/gcc_3/server_spa" > "$OUT/gcc_3/server_spa.objdump"
    python3 "$ATP/clean_objdump.py" "$OUT/gcc_3/server_spa.objdump" > "$OUT/gcc_3/server_spa.objdump_clean"
fi

if [ ! -f  "$OUT/gcc_3/server_default.objdump_clean" ]; then
    gcc -g -z execstack -no-pie  -o "$OUT/gcc_3/server_default" "$SRC/buffer_overflow/C/server.c"
    objdump -M intel --show-raw-insn -d "$OUT/gcc_3/server_default" > "$OUT/gcc_3/server_default.objdump"
    python3 "$ATP/clean_objdump.py" "$OUT/gcc_3/server_default.objdump" > "$OUT/gcc_3/server_default.objdump_clean"
fi

The option ( -M intel ) forces the intel convention in the disasssembling process. I prefer the intel convetion over the ATT convention.

1. The stack based buffer overflow mechanism

A "buffer overflow" occurs when a program writes more data to a buffer than its allocated capacity, overwriting adjacent memory and resulting in the corruption of adjacent memory —most notably the saved return address on the stack. It can happen when using functions like strcpy, memcpy, ..
Buffers can be allocated on stack so we use the term stack based buffer overflow or on heap and we use the term Heap buffer overflow.

For example in our C program, we copy the information found in request (256 bytes) in a buffer containing only 64 bytes using the following line
strcpy(buffer, request);
This results to a corruption of the stack which normally contains the return address of the function, so an attacker by modifying the return address can change the execution flow of the program and execute an arbitrary code.

Since this buffer is allocated on stack using a local variable, we are creating a stack based buffer overflow.

GCC default options partially protect programs from it but we will see that some stack based buffer overflows are still found each year even in programs where we do not expect to find them.

but let's start with GCC options

2. -fno-stack-protector

This option removes the default GCC mechanism of protecting programs from stack based buffer overflows. This is only useful to study the stack based buffer overflow mechanism and I don't see any other reason to have this option set when compiling a program.

0000000000401256 <test1>:
401256:  endbr64
40125a:  push   rbp
40125b:  mov    rbp,rsp
40125e:  sub    rsp,0x10
401262:  lea    rax,[rip+0xd9f]        # 402008 <_IO_stdin_used+0x8>
401269:  mov    QWORD PTR [rbp-0x8],rax
40126d:  lea    rax,[rip+0xffffffffffffffe2]        # 401256 <test1>
401274:  mov    rdi,rax
401277:  call   4010f0 <puts@plt>
40127c:  nop
40127d:  leave
40127e:  ret

000000000040127f <test2>:
40127f:  endbr64
401283:  push   rbp
401284:  mov    rbp,rsp
401287:  sub    rsp,0x10
40128b:  lea    rax,[rip+0xd7c]        # 40200e <_IO_stdin_used+0xe>
401292:  mov    BYTE PTR [rbp-0x1],al
401295:  lea    rax,[rip+0xd78]        # 402014 <_IO_stdin_used+0x14>
40129c:  mov    BYTE PTR [rbp-0x2],al
40129f:  movsx  edx,BYTE PTR [rbp-0x2]
4012a3:  movsx  eax,BYTE PTR [rbp-0x1]
4012a7:  mov    esi,eax
4012a9:  lea    rax,[rip+0xd66]        # 402016 <_IO_stdin_used+0x16>
4012b0:  mov    rdi,rax
4012b3:  mov    eax,0x0
4012b8:  call   401110 <printf@plt>
4012bd:  nop
4012be:  leave
4012bf:  ret

00000000004012c0 <handle_client>:
4012c0:  endbr64
4012c4:  push   rbp
4012c5:  mov    rbp,rsp
4012c8:  sub    rsp,0x160
4012cf:  mov    DWORD PTR [rbp-0x154],edi
4012d5:  lea    rsi,[rbp-0x150]
4012dc:  mov    eax,DWORD PTR [rbp-0x154]
4012e2:  mov    ecx,0x0
4012e7:  mov    edx,0xff
4012ec:  mov    edi,eax
4012ee:  call   4010d0 <recv@plt>
4012f3:  mov    DWORD PTR [rbp-0x4],eax
4012f6:  cmp    DWORD PTR [rbp-0x4],0x0
4012fa:  jle    40134e <handle_client+0x8e>
4012fc:  mov    eax,DWORD PTR [rbp-0x4]
4012ff:  cdqe
401301:  mov    BYTE PTR [rbp+rax*1-0x150],0x0
401309:  lea    rdx,[rbp-0x150]
401310:  lea    rax,[rbp-0x50]
401314:  mov    rsi,rdx
401317:  mov    rdi,rax
40131a:  call   4010e0 <strcpy@plt>
40131f:  lea    rax,[rbp-0x50]
401323:  mov    rsi,rax
401326:  lea    rax,[rip+0xcee]        # 40201b <_IO_stdin_used+0x1b>
40132d:  mov    rdi,rax
401330:  mov    eax,0x0
401335:  call   401110 <printf@plt>
40133a:  mov    eax,0x0
40133f:  call   401256 <test1>
401344:  mov    eax,0x0
401349:  call   40127f <test2>
40134e:  mov    eax,DWORD PTR [rbp-0x154]
401354:  mov    edi,eax
401356:  call   401120 <close@plt>
40135b:  nop
40135c:  leave
40135d:  ret

000000000040135e <main>:
40135e:  endbr64
401362:  push   rbp
401363:  mov    rbp,rsp
401366:  sub    rsp,0x20
40136a:  mov    edx,0x0
40136f:  mov    esi,0x1
401374:  mov    edi,0x2
401379:  call   401160 <socket@plt>
40137e:  mov    DWORD PTR [rbp-0x4],eax
401381:  mov    QWORD PTR [rbp-0x20],0x0
401389:  mov    QWORD PTR [rbp-0x18],0x0
401391:  mov    WORD PTR [rbp-0x20],0x2
401397:  mov    edi,0x1f90
40139c:  call   401100 <htons@plt>
4013a1:  mov    WORD PTR [rbp-0x1e],ax
4013a5:  lea    rcx,[rbp-0x20]
4013a9:  mov    eax,DWORD PTR [rbp-0x4]
4013ac:  mov    edx,0x10
4013b1:  mov    rsi,rcx
4013b4:  mov    edi,eax
4013b6:  call   401140 <bind@plt>
4013bb:  mov    eax,DWORD PTR [rbp-0x4]
4013be:  mov    esi,0x3
4013c3:  mov    edi,eax
4013c5:  call   401130 <listen@plt>
4013ca:  mov    esi,0x1f90
4013cf:  lea    rax,[rip+0xc5a]        # 402030 <_IO_stdin_used+0x30>
4013d6:  mov    rdi,rax
4013d9:  mov    eax,0x0
4013de:  call   401110 <printf@plt>
4013e3:  mov    eax,DWORD PTR [rbp-0x4]
4013e6:  mov    edx,0x0
4013eb:  mov    esi,0x0
4013f0:  mov    edi,eax
4013f2:  call   401150 <accept@plt>
4013f7:  mov    DWORD PTR [rbp-0x8],eax
4013fa:  mov    eax,DWORD PTR [rbp-0x8]
4013fd:  mov    edi,eax
4013ff:  call   4012c0 <handle_client>
401404:  nop
401405:  jmp    4013e3 <main+0x85>

let notice several things
First the size of the local memory allocated on the stack (0x160) for the handle_client function
4012c8: sub rsp,0x160

3. -fstack-protector

Let's see what the man page says about the option

-fstack-protector
Emit extra code to check for buffer overflows, such as stack
smashing attacks. This is done by adding a guard variable to
functions with vulnerable objects. This includes functions
that call "alloca", and functions with buffers larger than 8
bytes. The guards are initialized when a function is entered
and then checked when the function exits. If a guard check
fails, an error message is printed and the program exits

0000000000401276 <test1>:
401276:  endbr64
40127a:  push   rbp
40127b:  mov    rbp,rsp
40127e:  sub    rsp,0x10
401282:  lea    rax,[rip+0xd7f]        # 402008 <_IO_stdin_used+0x8>
401289:  mov    QWORD PTR [rbp-0x8],rax
40128d:  lea    rax,[rip+0xffffffffffffffe2]        # 401276 <test1>
401294:  mov    rdi,rax
401297:  call   401100 <puts@plt>
40129c:  nop
40129d:  leave
40129e:  ret

000000000040129f <test2>:
40129f:  endbr64
4012a3:  push   rbp
4012a4:  mov    rbp,rsp
4012a7:  sub    rsp,0x10
4012ab:  lea    rax,[rip+0xd5c]        # 40200e <_IO_stdin_used+0xe>
4012b2:  mov    BYTE PTR [rbp-0x2],al
4012b5:  lea    rax,[rip+0xd58]        # 402014 <_IO_stdin_used+0x14>
4012bc:  mov    BYTE PTR [rbp-0x1],al
4012bf:  movsx  edx,BYTE PTR [rbp-0x1]
4012c3:  movsx  eax,BYTE PTR [rbp-0x2]
4012c7:  mov    esi,eax
4012c9:  lea    rax,[rip+0xd46]        # 402016 <_IO_stdin_used+0x16>
4012d0:  mov    rdi,rax
4012d3:  mov    eax,0x0
4012d8:  call   401130 <printf@plt>
4012dd:  nop
4012de:  leave
4012df:  ret

00000000004012e0 <handle_client>:
4012e0:  endbr64
4012e4:  push   rbp
4012e5:  mov    rbp,rsp
4012e8:  sub    rsp,0x170
4012ef:  mov    DWORD PTR [rbp-0x164],edi
4012f5:  mov    rax,QWORD PTR fs:0x28
4012fe:  mov    QWORD PTR [rbp-0x8],rax
401302:  xor    eax,eax
401304:  lea    rsi,[rbp-0x110]
40130b:  mov    eax,DWORD PTR [rbp-0x164]
401311:  mov    ecx,0x0
401316:  mov    edx,0xff
40131b:  mov    edi,eax
40131d:  call   4010e0 <recv@plt>
401322:  mov    DWORD PTR [rbp-0x154],eax
401328:  cmp    DWORD PTR [rbp-0x154],0x0
40132f:  jle    40138c <handle_client+0xac>
401331:  mov    eax,DWORD PTR [rbp-0x154]
401337:  cdqe
401339:  mov    BYTE PTR [rbp+rax*1-0x110],0x0
401341:  lea    rdx,[rbp-0x110]
401348:  lea    rax,[rbp-0x150]
40134f:  mov    rsi,rdx
401352:  mov    rdi,rax
401355:  call   4010f0 <strcpy@plt>
40135a:  lea    rax,[rbp-0x150]
401361:  mov    rsi,rax
401364:  lea    rax,[rip+0xcb0]        # 40201b <_IO_stdin_used+0x1b>
40136b:  mov    rdi,rax
40136e:  mov    eax,0x0
401373:  call   401130 <printf@plt>
401378:  mov    eax,0x0
40137d:  call   401276 <test1>
401382:  mov    eax,0x0
401387:  call   40129f <test2>
40138c:  mov    eax,DWORD PTR [rbp-0x164]
401392:  mov    edi,eax
401394:  call   401140 <close@plt>
401399:  nop
40139a:  mov    rax,QWORD PTR [rbp-0x8]
40139e:  sub    rax,QWORD PTR fs:0x28
4013a7:  je     4013ae <handle_client+0xce>
4013a9:  call   401110 <__stack_chk_fail@plt>
4013ae:  leave
4013af:  ret

In this assembly , one can see that the option -fstack-protector has modified several things,

First (step 1) the size of the local memory allocated on the stack (0x170)
4012e8: sub rsp,0x170

Second (step 2) at the begining of the handle_function, right after the local memory allocation, GCC adds the following 3 lines
4012f5: mov rax,QWORD PTR fs:0x28
4012fe: mov QWORD PTR [rbp-0x8],rax
401302: xor eax,eax

Third (step 3) GCC adds some lines at the end of the handle_client function
40139a: mov rax,QWORD PTR [rbp-0x8]
40139e: sub rax,QWORD PTR fs:0x28
4013a7: je 4013ae
4013a9: call 401110 <__stack_chk_fail@plt>

So how this mechanism works ?
The program at execution is going to add a random list of bytes (called the stack canary, stack cookie ) at the end of the stack allocated.
In order to do that, it has to allocate some more space (step 1) on the stack than when there is no protection.
Then it copies 8 bytes from a specific location ( fs:0x28 see step 2) to the end of the stack ( [rbp-0x8] )

Then at the end of the function, the program checks that the stack canary has not been corrupted else it calls __stack_chk_fail@plt
Understanding mov rax, QWORD PTR fs:0x28
This instruction is used to ensure the integrity of the program's execution.
This instruction copies a value located in a special memory segment ( fs ) into the rax register.
In protected mode, the fs segment register points to a thread-specific memory structure called Thread Local Storage (TLS) or the tcbhead_t (Thread Control Block). The 0x28 offset: This is the specific location where the "Stack Canary" is stored.
Who generates the value at fs:0x28?
The execution environment, specifically the dynamic linker (/usr/lib64/ld-linux-x86-64.so.2 or /usr/lib32/ld-linux.so.2).

Here is the chain of responsibility:

a. The role of the Dynamic Linker
When you run a program on Linux, before the main function executes, the kernel loads the binary into memory and then hands control to the dynamic linker. This is the component that sets up the process environment.

During this initialization phase, the linker generates a random (or pseudo-random) value using a system call, typically getrandom() or by reading /dev/urandom. This value is then copied into the internal tcbhead_t structure associated with the fs segment. This is precisely when fs:0x28 is populated.

b. Why the linker and not the program?
It is imperative that this value is generated before your program code begins to execute. If the program itself generated the canary, it would need to be protected during that generation, creating a "chicken and egg" problem. By delegating this to the system linker, we ensure that every thread is protected from its very first cycle of execution.

c. Persistence of the value
Once the linker places this value at fs:0x28, it remains read-only (in theory) for the thread for its entire lifetime.

Per process: The canary is generally the same for all threads within a single process.

Per execution: Every time the program starts, the linker generates a new value. This is why even if an attacker discovers the canary for one session, they cannot use it for another; they would have to leak it again.

why the name stack canary ?
The name comes from the use of canaries in a mine. In mines they used canaries to alert for gas poisoning. If the canary stops singing, it means that you have to get out before you die from gas poisoning.
Finally
In the example above handle_client is protected but not test1 nor test2

4. fstack-protector-strong

Let's see what the man page says about the option
-fstack-protector-strong
Like -fstack-protector but includes additional functions to be
protected --- those that have local array definitions, or have
references to local frame addresses.

0000000000401276 <test1>:
401276:  endbr64
40127a:  push   rbp
40127b:  mov    rbp,rsp
40127e:  sub    rsp,0x10
401282:  lea    rax,[rip+0xd7f]        # 402008 <_IO_stdin_used+0x8>
401289:  mov    QWORD PTR [rbp-0x8],rax
40128d:  mov    rax,QWORD PTR [rbp-0x8]
401291:  mov    rdi,rax
401294:  call   401100 <puts@plt>
401299:  nop
40129a:  leave
40129b:  ret

000000000040129c <test2>:
40129c:  endbr64
4012a0:  push   rbp
4012a1:  mov    rbp,rsp
4012a4:  sub    rsp,0x70
4012a8:  mov    rax,QWORD PTR fs:0x28
4012b1:  mov    QWORD PTR [rbp-0x8],rax
4012b5:  xor    eax,eax
4012b7:  lea    rax,[rip+0xd50]        # 40200e <_IO_stdin_used+0xe>
4012be:  mov    BYTE PTR [rbp-0x61],al
4012c1:  movsx  rax,BYTE PTR [rbp-0x61]
4012c6:  mov    rdx,rax
4012c9:  mov    rax,QWORD PTR [rbp-0x60]
4012cd:  mov    rsi,rdx
4012d0:  mov    rdi,rax
4012d3:  call   4010f0 <strcpy@plt>
4012d8:  mov    rax,QWORD PTR [rbp-0x60]
4012dc:  mov    rsi,rax
4012df:  lea    rax,[rip+0xd2f]        # 402015 <_IO_stdin_used+0x15>
4012e6:  mov    rdi,rax
4012e9:  mov    eax,0x0
4012ee:  call   401130 <printf@plt>
4012f3:  nop
4012f4:  mov    rax,QWORD PTR [rbp-0x8]
4012f8:  sub    rax,QWORD PTR fs:0x28
401301:  je     401308 <test2+0x6c>
401303:  call   401110 <__stack_chk_fail@plt>
401308:  leave
401309:  ret

000000000040130a <handle_client>:
40130a:  endbr64
40130e:  push   rbp
40130f:  mov    rbp,rsp
401312:  sub    rsp,0x170
401319:  mov    DWORD PTR [rbp-0x164],edi
40131f:  mov    rax,QWORD PTR fs:0x28
401328:  mov    QWORD PTR [rbp-0x8],rax
40132c:  xor    eax,eax
40132e:  lea    rsi,[rbp-0x110]
401335:  mov    eax,DWORD PTR [rbp-0x164]
40133b:  mov    ecx,0x0
401340:  mov    edx,0xff
401345:  mov    edi,eax
401347:  call   4010e0 <recv@plt>
40134c:  mov    DWORD PTR [rbp-0x154],eax
401352:  cmp    DWORD PTR [rbp-0x154],0x0
401359:  jle    4013b6 <handle_client+0xac>
40135b:  mov    eax,DWORD PTR [rbp-0x154]
401361:  cdqe
401363:  mov    BYTE PTR [rbp+rax*1-0x110],0x0
40136b:  lea    rdx,[rbp-0x110]
401372:  lea    rax,[rbp-0x150]
401379:  mov    rsi,rdx
40137c:  mov    rdi,rax
40137f:  call   4010f0 <strcpy@plt>
401384:  lea    rax,[rbp-0x150]
40138b:  mov    rsi,rax
40138e:  lea    rax,[rip+0xc83]        # 402018 <_IO_stdin_used+0x18>
401395:  mov    rdi,rax
401398:  mov    eax,0x0
40139d:  call   401130 <printf@plt>
4013a2:  mov    eax,0x0
4013a7:  call   401276 <test1>
4013ac:  mov    eax,0x0
4013b1:  call   40129c <test2>
4013b6:  mov    eax,DWORD PTR [rbp-0x164]
4013bc:  mov    edi,eax
4013be:  call   401140 <close@plt>
4013c3:  nop
4013c4:  mov    rax,QWORD PTR [rbp-0x8]
4013c8:  sub    rax,QWORD PTR fs:0x28
4013d1:  je     4013d8 <handle_client+0xce>
4013d3:  call   401110 <__stack_chk_fail@plt>
4013d8:  leave
4013d9:  ret

As you can see here, handle_client , test2 are protected but not test1

5. fstack-protector-all

Let's see what the man page says about the option
-fstack-protector-all
Like -fstack-protector except that all functions are
protected.

0000000000401276 <test1>:
401276:  endbr64
40127a:  push   rbp
40127b:  mov    rbp,rsp
40127e:  sub    rsp,0x10
401282:  mov    rax,QWORD PTR fs:0x28
40128b:  mov    QWORD PTR [rbp-0x8],rax
40128f:  xor    eax,eax
401291:  lea    rax,[rip+0xd70]        # 402008 <_IO_stdin_used+0x8>
401298:  mov    QWORD PTR [rbp-0x10],rax
40129c:  mov    rax,QWORD PTR [rbp-0x10]
4012a0:  mov    rdi,rax
4012a3:  call   401100 <puts@plt>
4012a8:  nop
4012a9:  mov    rax,QWORD PTR [rbp-0x8]
4012ad:  sub    rax,QWORD PTR fs:0x28
4012b6:  je     4012bd <test1+0x47>
4012b8:  call   401110 <__stack_chk_fail@plt>
4012bd:  leave
4012be:  ret

00000000004012bf <test2>:
4012bf:  endbr64
4012c3:  push   rbp
4012c4:  mov    rbp,rsp
4012c7:  sub    rsp,0x70
4012cb:  mov    rax,QWORD PTR fs:0x28
4012d4:  mov    QWORD PTR [rbp-0x8],rax
4012d8:  xor    eax,eax
4012da:  lea    rax,[rip+0xd2d]        # 40200e <_IO_stdin_used+0xe>
4012e1:  mov    BYTE PTR [rbp-0x61],al
4012e4:  movsx  rax,BYTE PTR [rbp-0x61]
4012e9:  mov    rdx,rax
4012ec:  mov    rax,QWORD PTR [rbp-0x60]
4012f0:  mov    rsi,rdx
4012f3:  mov    rdi,rax
4012f6:  call   4010f0 <strcpy@plt>
4012fb:  movsx  eax,BYTE PTR [rbp-0x61]
4012ff:  mov    esi,eax
401301:  lea    rax,[rip+0xd0d]        # 402015 <_IO_stdin_used+0x15>
401308:  mov    rdi,rax
40130b:  mov    eax,0x0
401310:  call   401130 <printf@plt>
401315:  nop
401316:  mov    rax,QWORD PTR [rbp-0x8]
40131a:  sub    rax,QWORD PTR fs:0x28
401323:  je     40132a <test2+0x6b>
401325:  call   401110 <__stack_chk_fail@plt>
40132a:  leave
40132b:  ret

000000000040132c <handle_client>:
40132c:  endbr64
401330:  push   rbp
401331:  mov    rbp,rsp
401334:  sub    rsp,0x170
40133b:  mov    DWORD PTR [rbp-0x164],edi
401341:  mov    rax,QWORD PTR fs:0x28
40134a:  mov    QWORD PTR [rbp-0x8],rax
40134e:  xor    eax,eax
401350:  lea    rsi,[rbp-0x110]
401357:  mov    eax,DWORD PTR [rbp-0x164]
40135d:  mov    ecx,0x0
401362:  mov    edx,0xff
401367:  mov    edi,eax
401369:  call   4010e0 <recv@plt>
40136e:  mov    DWORD PTR [rbp-0x154],eax
401374:  cmp    DWORD PTR [rbp-0x154],0x0
40137b:  jle    4013d8 <handle_client+0xac>
40137d:  mov    eax,DWORD PTR [rbp-0x154]
401383:  cdqe
401385:  mov    BYTE PTR [rbp+rax*1-0x110],0x0
40138d:  lea    rdx,[rbp-0x110]
401394:  lea    rax,[rbp-0x150]
40139b:  mov    rsi,rdx
40139e:  mov    rdi,rax
4013a1:  call   4010f0 <strcpy@plt>
4013a6:  lea    rax,[rbp-0x150]
4013ad:  mov    rsi,rax
4013b0:  lea    rax,[rip+0xc63]        # 40201a <_IO_stdin_used+0x1a>
4013b7:  mov    rdi,rax
4013ba:  mov    eax,0x0
4013bf:  call   401130 <printf@plt>
4013c4:  mov    eax,0x0
4013c9:  call   401276 <test1>
4013ce:  mov    eax,0x0
4013d3:  call   4012bf <test2>
4013d8:  mov    eax,DWORD PTR [rbp-0x164]
4013de:  mov    edi,eax
4013e0:  call   401140 <close@plt>
4013e5:  nop
4013e6:  mov    rax,QWORD PTR [rbp-0x8]
4013ea:  sub    rax,QWORD PTR fs:0x28
4013f3:  je     4013fa <handle_client+0xce>
4013f5:  call   401110 <__stack_chk_fail@plt>
4013fa:  leave
4013fb:  ret

As you can see here, handle_client , test2 and test1 are protected

6. Default option

0000000000401276 <test1>:
401276:  endbr64
40127a:  push   rbp
40127b:  mov    rbp,rsp
40127e:  sub    rsp,0x10
401282:  lea    rax,[rip+0xd7f]        # 402008 <_IO_stdin_used+0x8>
401289:  mov    QWORD PTR [rbp-0x8],rax
40128d:  mov    rax,QWORD PTR [rbp-0x8]
401291:  mov    rdi,rax
401294:  call   401100 <puts@plt>
401299:  nop
40129a:  leave
40129b:  ret

000000000040129c <test2>:
40129c:  endbr64
4012a0:  push   rbp
4012a1:  mov    rbp,rsp
4012a4:  sub    rsp,0x70
4012a8:  mov    rax,QWORD PTR fs:0x28
4012b1:  mov    QWORD PTR [rbp-0x8],rax
4012b5:  xor    eax,eax
4012b7:  lea    rax,[rip+0xd50]        # 40200e <_IO_stdin_used+0xe>
4012be:  mov    BYTE PTR [rbp-0x61],al
4012c1:  movsx  rax,BYTE PTR [rbp-0x61]
4012c6:  mov    rdx,rax
4012c9:  mov    rax,QWORD PTR [rbp-0x60]
4012cd:  mov    rsi,rdx
4012d0:  mov    rdi,rax
4012d3:  call   4010f0 <strcpy@plt>
4012d8:  mov    rax,QWORD PTR [rbp-0x60]
4012dc:  mov    rsi,rax
4012df:  lea    rax,[rip+0xd2f]        # 402015 <_IO_stdin_used+0x15>
4012e6:  mov    rdi,rax
4012e9:  mov    eax,0x0
4012ee:  call   401130 <printf@plt>
4012f3:  nop
4012f4:  mov    rax,QWORD PTR [rbp-0x8]
4012f8:  sub    rax,QWORD PTR fs:0x28
401301:  je     401308 <test2+0x6c>
401303:  call   401110 <__stack_chk_fail@plt>
401308:  leave
401309:  ret

000000000040130a <handle_client>:
40130a:  endbr64
40130e:  push   rbp
40130f:  mov    rbp,rsp
401312:  sub    rsp,0x170
401319:  mov    DWORD PTR [rbp-0x164],edi
40131f:  mov    rax,QWORD PTR fs:0x28
401328:  mov    QWORD PTR [rbp-0x8],rax
40132c:  xor    eax,eax
40132e:  lea    rsi,[rbp-0x110]
401335:  mov    eax,DWORD PTR [rbp-0x164]
40133b:  mov    ecx,0x0
401340:  mov    edx,0xff
401345:  mov    edi,eax
401347:  call   4010e0 <recv@plt>
40134c:  mov    DWORD PTR [rbp-0x154],eax
401352:  cmp    DWORD PTR [rbp-0x154],0x0
401359:  jle    4013b6 <handle_client+0xac>
40135b:  mov    eax,DWORD PTR [rbp-0x154]
401361:  cdqe
401363:  mov    BYTE PTR [rbp+rax*1-0x110],0x0
40136b:  lea    rdx,[rbp-0x110]
401372:  lea    rax,[rbp-0x150]
401379:  mov    rsi,rdx
40137c:  mov    rdi,rax
40137f:  call   4010f0 <strcpy@plt>
401384:  lea    rax,[rbp-0x150]
40138b:  mov    rsi,rax
40138e:  lea    rax,[rip+0xc83]        # 402018 <_IO_stdin_used+0x18>
401395:  mov    rdi,rax
401398:  mov    eax,0x0
40139d:  call   401130 <printf@plt>
4013a2:  mov    eax,0x0
4013a7:  call   401276 <test1>
4013ac:  mov    eax,0x0
4013b1:  call   40129c <test2>
4013b6:  mov    eax,DWORD PTR [rbp-0x164]
4013bc:  mov    edi,eax
4013be:  call   401140 <close@plt>
4013c3:  nop
4013c4:  mov    rax,QWORD PTR [rbp-0x8]
4013c8:  sub    rax,QWORD PTR fs:0x28
4013d1:  je     4013d8 <handle_client+0xce>
4013d3:  call   401110 <__stack_chk_fail@plt>
4013d8:  leave
4013d9:  ret

As we can see above, the default GCC option seems to correspond to -fstack-protector-strong

7. Some history

The -fstack-protector option was introduced into GCC in 2005, specifically with the release of GCC 4.1.

Pre-2005: Before this native GCC integration, developers who wanted stack protection had to rely on a patch for GCC called "StackGuard" (originally developed by Crispin Cowan and his team at the Oregon Graduate Institute in the late 1990s). StackGuard was the pioneering implementation that popularized the "canary" concept.

2005 (GCC 4.1): GCC officially integrated the functionality, making it a standard compiler feature rather than a third-party patch. This was a major milestone for security because it made hardening binaries accessible to the entire open-source ecosystem without requiring custom compiler builds.

Evolution (GCC 4.9+): While the original -fstack-protector existed in 4.1, GCC later introduced more granular levels to balance security and performance:

-fstack-protector-strong (added in GCC 4.9, 2014): Protects functions that have local arrays or references to local stack variables, providing a better middle ground.

-fstack-protector-all: The "paranoid" mode that protects every single function, regardless of its local variables.

8. The situation now

With the -fstack-protector-strong option activated by default, we should not find stack buffer overflows in the CVE database. However we still find a lot of them.

if we search for "Stack-based Buffer Overflow Remote Code Execution" we still find 437 CVE like the following one

Updated: 2026-03-19

Published: 2026-02-02
Title: Libsoup: stack-based buffer overflow in libsoup multipart response parsingmultipart http response

Description
A flaw was found in libsoup. This stack-based buffer overflow vulnerability occurs during the parsing of multipart HTTP responses due to an incorrect length calculation. A remote attacker can exploit this by sending a specially crafted multipart HTTP response, which can lead to memory corruption. This issue may result in application crashes or arbitrary code execution in applications that process untrusted server responses, and it does not require authentication or user interaction.

if we search for "Stack-based Buffer Overflow" we find 5809 CVE like the following one

For example

Updated: 2026-06-29

Published: 2026-05-20
Title: Libsolv: stack-based buffer overflow in libsolv's debian metadata parser when handling sha384/sha512 checksums

Description
A flaw was found in libsolv. This stack-based buffer overflow vulnerability occurs in libsolv's Debian metadata parser when processing specially crafted Debian repository metadata. An attacker could exploit this by providing malicious SHA384 or SHA512 checksum tags, leading to memory corruption and a denial of service (DoS) in the affected system.

Even when a function is hardened with a stack canary, it remains blind to vulnerabilities occurring within the heap or logic errors that corrupt data pointers before the canary check is ever reached

if we search for "Buffer Overflow" we find 22018 CVE most of them being heap buffer overflows

9. Why so ?

The "Non-Stack" Overflows The canary only protects the Return Address on the stack. It does nothing to stop overflows that target other critical data structures:

Heap Overflows: If the buffer is allocated via malloc, the canary is not involved. Attackers target metadata (like chunk headers in glibc) or function pointers stored in the heap to hijack execution.

Global Data Overflows: Overwriting global variables, pointers, or flags in the .data or .bss segments doesn't trigger the stack canary.

Struct Overwriting: If you have two local variables (e.g., a buffer followed by a function pointer) on the stack, an attacker can overflow the buffer to overwrite the pointer before reaching the canary. The function will use the corrupted pointer, and the program will never reach the "canary check" epilogue.

The Legacy Burden (The "Unprotected" World) Millions of lines of C/C++ code: Much of the critical infrastructure (drivers, legacy embedded systems, older enterprise software) was written long before 2014. Migrating this code to compile with modern security flags is a massive, risky, and often prohibitively expensive task.

Non-GCC Compilers: Many specialized environments (embedded firmware, proprietary real-time operating systems) use custom or older compilers that do not have stack-protector-strong implemented or enabled by default.

10. Epilogue

By disassembling the binary (with objdump), one precisely see what gcc creates during the compilation process. In this article, I explained the main options one can use to protect a program from a buffer overflow attack. There are several other options (-fstack-protector-explicit, -fstack-check, -fstack-clash-protection) to precisely manage the response to a stack smash attack.

Even if the default GCC option protects programs compiled with GCC from stack buffer overflows, there are still plenty of them found in the CVE database which shows that we can't rely on default protections to feel secure.

In the next articles of this series, I will continue to study the options GCC offers to protect a binary from attackers.

MIPS From Source Code to Binary: What Does GCC Actually Do ?

ddupard — Fri, 17 Jul 2026 08:28:38 +0000

Comparing C source code with its machine-level translation is an invaluable exercise for any developer or security researcher. Compilation is not a mere literal translation; it is an optimization process where the compiler often prioritizes portability and robustness over the use of direct, specialized processor instructions. To illustrate this, we are going to use a simple program which will be compiled for a MIPS processor:

#include <stdio.h>
#include <math.h>



void test_floating_point(float f) {
    // C'est ici qu'on verra si le compilateur génère 'abs.s'
    float res = fabsf(f);
    printf("FABS of %f is %f\n", f, res);
}

int main() {

    test_floating_point(-3.14159f);

    return 0;
}

The commands to compile this program are the following

  mips-linux-gnu-gcc -g -fno-stack-protector -z execstack -no-pie -march=mips32 -mabi=32 -o "$OUT_MIPS/tests/mips1" "$SRC/mips1.c"
  mips-linux-gnu-objdump -d --show-raw-insn -M reg-names=32 "$OUT_MIPS/tests/mips1" > "$OUT_MIPS/tests/mips1.mips"
  mips-linux-gnu-readelf -a "$OUT_MIPS/tests/mips1" > "$OUT_MIPS/tests/mips1.mips_elf"

and below the assembly

00400650 <test_floating_point>:
  400650:   27bdffc8    addiu   sp,sp,-56
  400654:   afbf0034    sw  ra,52(sp)
  400658:   afbe0030    sw  s8,48(sp)
  40065c:   03a0f025    move    s8,sp
  400660:   3c1c0043    lui gp,0x43
  400664:   279c8010    addiu   gp,gp,-32752
  400668:   afbc0018    sw  gp,24(sp)
  40066c:   e7cc0038    swc1    fa0,56(s8)
  400670:   8fc30038    lw  v1,56(s8)
  400674:   3c027fff    lui v0,0x7fff
  400678:   3442ffff    ori v0,v0,0xffff
  40067c:   00621024    and v0,v1,v0
  400680:   afc20020    sw  v0,32(s8)
  400684:   c7c00038    lwc1    fv0,56(s8)
  400688:   460000a1    cvt.d.s fv1,fv0
  40068c:   c7c00020    lwc1    fv0,32(s8)
  400690:   46000021    cvt.d.s fv0,fv0
  400694:   f7a00010    sdc1    fv0,16(sp)
  400698:   f7c20028    sdc1    fv1,40(s8)
  40069c:   8fc7002c    lw  a3,44(s8)
  4006a0:   8fc60028    lw  a2,40(s8)
  4006a4:   3c020040    lui v0,0x40
  4006a8:   24440790    addiu   a0,v0,1936
  4006ac:   8f828024    lw  v0,-32732(gp)
  4006b0:   0040c825    move    t9,v0
  4006b4:   0320f809    jalr    t9
  4006b8:   00000000    nop
  4006bc:   8fdc0018    lw  gp,24(s8)
  4006c0:   00000000    nop
  4006c4:   03c0e825    move    sp,s8
  4006c8:   8fbf0034    lw  ra,52(sp)
  4006cc:   8fbe0030    lw  s8,48(sp)
  4006d0:   27bd0038    addiu   sp,sp,56
  4006d4:   03e00008    jr  ra
  4006d8:   00000000    nop

004006dc <main>:
  4006dc:   27bdffe0    addiu   sp,sp,-32
  4006e0:   afbf001c    sw  ra,28(sp)
  4006e4:   afbe0018    sw  s8,24(sp)
  4006e8:   03a0f025    move    s8,sp
  4006ec:   3c020040    lui v0,0x40
  4006f0:   c44007a4    lwc1    fv0,1956(v0)
  4006f4:   46000306    mov.s   fa0,fv0
  4006f8:   0c100194    jal 400650 <test_floating_point>
  4006fc:   00000000    nop
  400700:   00001025    move    v0,zero
  400704:   03c0e825    move    sp,s8
  400708:   8fbf001c    lw  ra,28(sp)
  40070c:   8fbe0018    lw  s8,24(sp)
  400710:   27bd0020    addiu   sp,sp,32
  400714:   03e00008    jr  ra
  400718:   00000000    nop
  40071c:   00000000    nop

Disassembly of section .MIPS.stubs:

00400720 <_MIPS_STUBS_>:
  400720:   8f998010    lw  t9,-32752(gp)
  400724:   03e07825    move    t7,ra
  400728:   0320f809    jalr    t9
  40072c:   24180009    li  t8,9
  400730:   8f998010    lw  t9,-32752(gp)
  400734:   03e07825    move    t7,ra
  400738:   0320f809    jalr    t9
  40073c:   24180007    li  t8,7
    ...

and a part of the elf

Global entries

Address	Access	Initial	Sym.Val.	Type	Ndx	Name
0042002c	-32740(gp)	00000000	00000000	NOTYPE	UND	_ITM_registerTMCloneTable
00420030	-32736(gp)	00000000	00000000	FUNC	UND	gmon_start
00420034	-32732(gp)	00400730	00400730	FUNC	UND	printf
00420038	-32728(gp)	00000000	00000000	NOTYPE	UND	_ITM_deregisterTMCloneTable
0042003c	-32724(gp)	00400720	00400720	FUNC	UND	__libc_start_main

1. why it is interesting ?

To understand, you have to know that the MIPS® Architecture For Programmers Volume II-A which is the reference for the instructions set of the mips processor contains very specific instructions to calculate the absolute value of a floating point. In fact it contains 3 specific instructions
ABS.S
ABS.D
ABS.PS

Instruction Set

so we should find one of these instructions in the test_floating_point function. we don't find any of these instructions, instead we find

400674: 3c027fff    lui v0,0x7fff       # v0 = 0x7fff0000
400678: 3442ffff    ori v0,v0,0xffff    # v0 = 0x7fffffff
40067c: 00621024    and v0,v1,v0        # v0 = v1 & 0x7fffffff

The compiler applied a fundamental property of the IEEE 754 format here: a 32-bit floating-point number consists of one sign bit, eight exponent bits, and 23 mantissa bits. To calculate the absolute value, it is sufficient to force the sign bit to 0. This is exactly what the AND operation with 0x7FFFFFFF performs.
As seen with this example, the compiler often generates code compatible with multiple FPU variants, or even "Soft-Float" modes, to avoid creating overly strict dependencies on specific hardware.

2. Epilogue

In this very short article we explained a very specific point of what gcc does when it compiles a C source code. This shows why disassembling a binary compiled with gcc is always interesting. As a reverser, this analysis teaches us a crucial lesson: never rely solely on an architecture's theoretical documentation. Real code is the result of a chemistry between the processor manual, the ABI (Application Binary Interface) rules, and the compiler's optimization choices.

X86_64 From Source Code to Binary: What Does GCC Actually Do? part 2

ddupard — Wed, 08 Jul 2026 22:32:40 +0000

In the last part of my first article on how GCC compiles, I explained the System V AMD64 ABI convention for passing parameters, it's time to explain how a function returns a result. But before that, I am going to explain the mechanism used by GCC to allocate some space in stack memory. Stack memory is a built in mechanism for x86 processors. It comes with specific instructions (push , pop ) to save and restore some data and a specific register rsp ( stack pointer register). It is used by compilers (like GCC) to store local data in functions

To illustrate the return mechanism and the stack memory allocation, I will use the following program:

#include <stdio.h>

// 1. Return via register (Standard scalar type)
int return_integer() {
    return 42;
}

// 2. Return via hidden pointer (Large struct)
// The caller provides a memory address where the result should be stored
typedef struct {
    long buffer[4];
} LargeData;

LargeData return_large_struct() {
    LargeData ld = {{1, 2, 3, 4}};
    return ld;
}


// 3. Return via multiple registers (Small struct optimization)
// On x86_64, GCC places 'first' in RAX and 'second' in RDX
typedef struct {
    long first;
    long second;
} PairOfLongs;

PairOfLongs return_pair() {
    PairOfLongs p = {100, 200};
    return p;
}


int main() {
    int i = return_integer();
    LargeData ld = return_large_struct();
    PairOfLongs p = return_pair();


    printf("Integer: %d\n", i);
    printf("Large: %ld, %ld\n", ld.buffer[0], ld.buffer[3]);
    printf("Pair: %ld, %ld\n", p.first, p.second);


    return 0;
}

This program will be compiled with GCC and the binary generated will be disassembled by objdump using the following commands

gcc -g -fno-stack-protector -z execstack -no-pie -o a3 ./a3.c
objdump -M intel --show-raw-insn -d a3 > a3.objdump

The option ( -M intel ) forces the intel convention in the disassembling process. I prefer the intel convetion over the ATT convention.

1. Stack memory allocation

Most of the functions compiled with gcc start with the same 4 lines (see below)

00000000004011bc <main>:
  4011bc:   f3 0f 1e fa             endbr64
  4011c0:   55                      push   rbp
  4011c1:   48 89 e5                mov    rbp,rsp
  4011c4:   48 83 ec 40             sub    rsp,0x40

endbr64 (end branch 64 bits) is a security checkpoint used by modern INTEL and AMD processors, It is part of a hardware protection technology called Control-flow Enforcement Technology (CET). It stops "Jump-Oriented Programming" (JOP): Attackers often try to redirect the flow of a program to existing snippets of code to do things they weren't designed to do. endbr64 prevents these unauthorized redirects. It’s a hardware-level gate because it is checked directly by the processor. We will come back on this subject in another article.

push rbp save the base pointer register to the stack

mov rbp,rsp stores the value of the stack pointer register to the base pointer register. By doing this we create a stack frame.

sub rsp,0x40 allocates a certain amount of memory for the variables of the function (here 0x40 = 64 bytes). GCC computes the necessary space for all the variables declared in the function and allocates some stack memory space by changing the value of the stack pointer register (rsp). One can save some data to the stack by the push instruction ( ex push rax ) and can retrieve data from the stack by using the pop instruction ( ex pop rax). push and pop modifies the stack pointer register (rsp), push by decreasing its value, and pop by increasing its value. The stack pointer register stores 64 bits values on a 64 bit OS, 32 bits values on 32 bits OS, etc.

2. How a function compiled by GCC returns some result

Depending on the size of the results returned, GCC uses one of the following 3 mechanisms:
1) the result is returned by a register (RAX, EAX, AX, AL)
2) the result is returned by a hidden pointer
3) the result is returned on several registers (for floating pointers)

3. Return by a register

0000000000401136 <return_integer>:
  401136:   f3 0f 1e fa             endbr64
  40113a:   55                      push   rbp
  40113b:   48 89 e5                mov    rbp,rsp
  40113e:   b8 2a 00 00 00          mov    eax,0x2a
  401143:   5d                      pop    rbp
  401144:   c3                      ret

00000000004011bc <main>:
  ...
  4011c8:   b8 00 00 00 00          mov    eax,0x0
  4011cd:   e8 64 ff ff ff          call   401136 <return_integer>
  4011d2:   89 45 fc                mov    DWORD PTR [rbp-0x4],eax

This is the most common return mechanism, a value is returned by a single register (here eax). Notice that even in a 64 bits OS, the value returned can be a 64 bits value, a 32 bits value, a 16 bits value or even a 8 bits value.

mov DWORD PTR [rbp-0x4],eax moves the value of the register eax (the result of the function return_integer) to a specific memory location in the stack

2. Return by a hidden pointer


0000000000401145 <return_large_struct>:
  401145:   f3 0f 1e fa             endbr64
  401149:   55                      push   rbp
  40114a:   48 89 e5                mov    rbp,rsp
  40114d:   48 89 7d d8             mov    QWORD PTR [rbp-0x28],rdi
  401151:   48 c7 45 e0 01 00 00    mov    QWORD PTR [rbp-0x20],0x1
  401158:   00 
  401159:   48 c7 45 e8 02 00 00    mov    QWORD PTR [rbp-0x18],0x2
  401160:   00 
  401161:   48 c7 45 f0 03 00 00    mov    QWORD PTR [rbp-0x10],0x3
  401168:   00 
  401169:   48 c7 45 f8 04 00 00    mov    QWORD PTR [rbp-0x8],0x4
  401170:   00 
  401171:   48 8b 4d d8             mov    rcx,QWORD PTR [rbp-0x28]
  401175:   48 8b 45 e0             mov    rax,QWORD PTR [rbp-0x20]
  401179:   48 8b 55 e8             mov    rdx,QWORD PTR [rbp-0x18]
  40117d:   48 89 01                mov    QWORD PTR [rcx],rax
  401180:   48 89 51 08             mov    QWORD PTR [rcx+0x8],rdx
  401184:   48 8b 45 f0             mov    rax,QWORD PTR [rbp-0x10]
  401188:   48 8b 55 f8             mov    rdx,QWORD PTR [rbp-0x8]
  40118c:   48 89 41 10             mov    QWORD PTR [rcx+0x10],rax
  401190:   48 89 51 18             mov    QWORD PTR [rcx+0x18],rdx
  401194:   48 8b 45 d8             mov    rax,QWORD PTR [rbp-0x28]
  401198:   5d                      pop    rbp
  401199:   c3                      ret


00000000004011bc <main>:
  ...
  4011d5:   48 8d 45 d0             lea    rax,[rbp-0x30]
  4011d9:   48 89 c7                mov    rdi,rax
  4011dc:   b8 00 00 00 00          mov    eax,0x0
  4011e1:   e8 5f ff ff ff          call   401145 <return_large_struct>

Because the result returned by the function is a large structure with a size far greater than 8 bytes, the register mechanism can't be used.
see what happens before the call, with the 2 instructions lea rax,[rbp-0x30] and mov rdi,rax rdi receives rbp-0x30 which corresponds
to an address in the stack. Even if the function return_large_struct has no parameter, rdi is filled with a value which is then used in the function return_large_struct.
Using RDI as a pointer is why this mechanism is called the hidden pointer mechanism.

In this mechanism, the called function performs zero stack allocation for the result. It treats the address provided in RDI as a raw memory pointer. The called function essentially
acts as a 'data mover,' writing its local results directly into the memory frame that was already reserved by the caller. And in our example the caller has reserved data in the stack.

The only "real" instructions of the function return_large_struct are the 4 following lines

  401151:   48 c7 45 e0 01 00 00    mov    QWORD PTR [rbp-0x20],0x1
  401158:   00 
  401159:   48 c7 45 e8 02 00 00    mov    QWORD PTR [rbp-0x18],0x2
  401160:   00 
  401161:   48 c7 45 f0 03 00 00    mov    QWORD PTR [rbp-0x10],0x3
  401168:   00 
  401169:   48 c7 45 f8 04 00 00    mov    QWORD PTR [rbp-0x8],0x4
  401170:   00

By default, objdump prioritizes human-readable assembly over raw opcode alignment. When an instruction's length causes a mismatch with the internal column formatting, it may wrap the remaining bytes to a new line, hence 00 on a new line after the instruction line. Note that the 00 bytes appearing on separate lines are not instructions, but rather the hexadecimal representation of the immediate data values encoded directly within the machine instructions.

The following lines have only one purpose: to return the information to the caller. See how the compiler uses the register rcx and the stack to store data

  40117d:   48 89 01                mov    QWORD PTR [rcx],rax
  401180:   48 89 51 08             mov    QWORD PTR [rcx+0x8],rdx
  401184:   48 8b 45 f0             mov    rax,QWORD PTR [rbp-0x10]
  401188:   48 8b 55 f8             mov    rdx,QWORD PTR [rbp-0x8]
  40118c:   48 89 41 10             mov    QWORD PTR [rcx+0x10],rax
  401190:   48 89 51 18             mov    QWORD PTR [rcx+0x18],rdx

3. Return by multiple registers

000000000040119a <return_pair>:
  40119a:   f3 0f 1e fa             endbr64
  40119e:   55                      push   rbp
  40119f:   48 89 e5                mov    rbp,rsp
  4011a2:   48 c7 45 f0 64 00 00    mov    QWORD PTR [rbp-0x10],0x64
  4011a9:   00 
  4011aa:   48 c7 45 f8 c8 00 00    mov    QWORD PTR [rbp-0x8],0xc8
  4011b1:   00 
  4011b2:   48 8b 45 f0             mov    rax,QWORD PTR [rbp-0x10]
  4011b6:   48 8b 55 f8             mov    rdx,QWORD PTR [rbp-0x8]
  4011ba:   5d                      pop    rbp
  4011bb:   c3                      ret

00000000004011bc <main>:
  ...
  4011e6:   b8 00 00 00 00          mov    eax,0x0
  4011eb:   e8 aa ff ff ff          call   40119a <return_pair>
  4011f0:   48 89 45 c0             mov    QWORD PTR [rbp-0x40],rax
  4011f4:   48 89 55 c8             mov    QWORD PTR [rbp-0x38],rdx

Here what the compiler does is pretty straightforward, it uses the registers rax and rdx to return the values to the caller function.

Right now , we've seen the 3 main ways used by gcc to return values from a function. but if we use some compilation options , we'll see that a lot can change.

4. what happens when we change some compilation options

Now we are going to use the following command to compile

gcc -g -fno-stack-protector -z execstack -no-pie -O3 -o a3_o3 ./a3.c
objdump -M intel --show-raw-insn -d a3_o3 > a3_o3.objdump

The result is really different from what we had previously


0000000000401050 <main>:
  401050:   f3 0f 1e fa             endbr64
  401054:   48 83 ec 08             sub    rsp,0x8
  401058:   ba 2a 00 00 00          mov    edx,0x2a
  40105d:   bf 02 00 00 00          mov    edi,0x2
  401062:   31 c0                   xor    eax,eax
  401064:   48 8d 35 99 0f 00 00    lea    rsi,[rip+0xf99]        # 402004 <_IO_stdin_used+0x4>
  40106b:   e8 d0 ff ff ff          call   401040 <__printf_chk@plt>
  401070:   b9 04 00 00 00          mov    ecx,0x4
  401075:   ba 01 00 00 00          mov    edx,0x1
  40107a:   31 c0                   xor    eax,eax
  40107c:   48 8d 35 8e 0f 00 00    lea    rsi,[rip+0xf8e]        # 402011 <_IO_stdin_used+0x11>
  401083:   bf 02 00 00 00          mov    edi,0x2
  401088:   e8 b3 ff ff ff          call   401040 <__printf_chk@plt>
  40108d:   b9 c8 00 00 00          mov    ecx,0xc8
  401092:   ba 64 00 00 00          mov    edx,0x64
  401097:   31 c0                   xor    eax,eax
  401099:   48 8d 35 82 0f 00 00    lea    rsi,[rip+0xf82]        # 402022 <_IO_stdin_used+0x22>
  4010a0:   bf 02 00 00 00          mov    edi,0x2
  4010a5:   e8 96 ff ff ff          call   401040 <__printf_chk@plt>
  4010aa:   31 c0                   xor    eax,eax
  4010ac:   48 83 c4 08             add    rsp,0x8
  4010b0:   c3                      ret
  4010b1:   66 2e 0f 1f 84 00 00    cs nop WORD PTR [rax+rax*1+0x0]
  4010b8:   00 00 00 
  4010bb:   0f 1f 44 00 00          nop    DWORD PTR [rax+rax*1+0x0]
...

00000000004011b0 <return_integer>:
  4011b0:   f3 0f 1e fa             endbr64
  4011b4:   b8 2a 00 00 00          mov    eax,0x2a
  4011b9:   c3                      ret
  4011ba:   66 0f 1f 44 00 00       nop    WORD PTR [rax+rax*1+0x0]

00000000004011c0 <return_large_struct>:
  4011c0:   f3 0f 1e fa             endbr64
  4011c4:   66 0f 6f 05 74 0e 00    movdqa xmm0,XMMWORD PTR [rip+0xe74]        # 402040 <_IO_stdin_used+0x40>
  4011cb:   00 
  4011cc:   48 89 f8                mov    rax,rdi
  4011cf:   0f 11 07                movups XMMWORD PTR [rdi],xmm0
  4011d2:   66 0f 6f 05 76 0e 00    movdqa xmm0,XMMWORD PTR [rip+0xe76]        # 402050 <_IO_stdin_used+0x50>
  4011d9:   00 
  4011da:   0f 11 47 10             movups XMMWORD PTR [rdi+0x10],xmm0
  4011de:   c3                      ret
  4011df:   90                      nop

00000000004011e0 <return_pair>:
  4011e0:   f3 0f 1e fa             endbr64
  4011e4:   b8 64 00 00 00          mov    eax,0x64
  4011e9:   ba c8 00 00 00          mov    edx,0xc8
  4011ee:   c3                      ret

There is no call of the different functions in the main function because the main already contains the result of all these functions.
With optimization (-O3): The compiler realizes, "Why call ? I'll just put the instructions for return_integer right here inside main.

So if there is no call and the results of the functions have been integrated directly in the main function why do we still find the 3 functions (return_integer, return_large_struct, return_pair) in the binary ?

GCC and the linker often preserve the original function code for a few critical reasons.

1. The "Export" Requirement (Symbol Visibility)

By default, in C, functions have external linkage. This means that another object file or a shared library might technically be able to call return_integer. Even if your current main doesn't use the function anymore, GCC cannot guarantee that no one else will, so it keeps the code in the binary to ensure the symbol remains resolvable.

2. Debugging and Profiling

If you compile with debug symbols (even if you don't use -g, some remnants remain), the linker tries to maintain a map between the source code functions and the machine code. If it deleted the function entirely, tools like gdb or perf (profilers) would not be able to "find" the function, making it impossible to step through or profile the original code.

3. The "Ghost" Code Phenomenon

Think of the binary as a storage container. When you compile:

GCC generates the object code for every function defined in a3.c.

The linker then combines these. If you haven't explicitly asked the linker to "garbage collect" unused code (a feature called Dead Code Elimination), it will simply place all generated code into the .text section of the final binary.

If you want a truly minimal binary where these "orphaned" functions are physically removed, you can use linker flags that perform Dead Code Elimination. You can add these flags to your compilation command:

gcc -O3 -ffunction-sections -fdata-sections -Wl,--gc-sections a3.c -o a3_min

5. Epilogue

By disassembling the binary (with objdump), one precisely see what gcc creates during the compilation process. In this article, I explained the stack allocation mechanism and the way GCC returns a result from a function and I showed that some compilations options could really change the binary code.

6. Disasm of the binary generated by GCC

Below you find the importants parts of the disassembly of the binary generated by gcc:

0000000000401136 <return_integer>:
  401136:   f3 0f 1e fa             endbr64
  40113a:   55                      push   rbp
  40113b:   48 89 e5                mov    rbp,rsp
  40113e:   b8 2a 00 00 00          mov    eax,0x2a
  401143:   5d                      pop    rbp
  401144:   c3                      ret

0000000000401145 <return_large_struct>:
  401145:   f3 0f 1e fa             endbr64
  401149:   55                      push   rbp
  40114a:   48 89 e5                mov    rbp,rsp
  40114d:   48 89 7d d8             mov    QWORD PTR [rbp-0x28],rdi
  401151:   48 c7 45 e0 01 00 00    mov    QWORD PTR [rbp-0x20],0x1
  401158:   00 
  401159:   48 c7 45 e8 02 00 00    mov    QWORD PTR [rbp-0x18],0x2
  401160:   00 
  401161:   48 c7 45 f0 03 00 00    mov    QWORD PTR [rbp-0x10],0x3
  401168:   00 
  401169:   48 c7 45 f8 04 00 00    mov    QWORD PTR [rbp-0x8],0x4
  401170:   00 
  401171:   48 8b 4d d8             mov    rcx,QWORD PTR [rbp-0x28]
  401175:   48 8b 45 e0             mov    rax,QWORD PTR [rbp-0x20]
  401179:   48 8b 55 e8             mov    rdx,QWORD PTR [rbp-0x18]
  40117d:   48 89 01                mov    QWORD PTR [rcx],rax
  401180:   48 89 51 08             mov    QWORD PTR [rcx+0x8],rdx
  401184:   48 8b 45 f0             mov    rax,QWORD PTR [rbp-0x10]
  401188:   48 8b 55 f8             mov    rdx,QWORD PTR [rbp-0x8]
  40118c:   48 89 41 10             mov    QWORD PTR [rcx+0x10],rax
  401190:   48 89 51 18             mov    QWORD PTR [rcx+0x18],rdx
  401194:   48 8b 45 d8             mov    rax,QWORD PTR [rbp-0x28]
  401198:   5d                      pop    rbp
  401199:   c3                      ret

000000000040119a <return_pair>:
  40119a:   f3 0f 1e fa             endbr64
  40119e:   55                      push   rbp
  40119f:   48 89 e5                mov    rbp,rsp
  4011a2:   48 c7 45 f0 64 00 00    mov    QWORD PTR [rbp-0x10],0x64
  4011a9:   00 
  4011aa:   48 c7 45 f8 c8 00 00    mov    QWORD PTR [rbp-0x8],0xc8
  4011b1:   00 
  4011b2:   48 8b 45 f0             mov    rax,QWORD PTR [rbp-0x10]
  4011b6:   48 8b 55 f8             mov    rdx,QWORD PTR [rbp-0x8]
  4011ba:   5d                      pop    rbp
  4011bb:   c3                      ret

00000000004011bc <main>:
  4011bc:   f3 0f 1e fa             endbr64
  4011c0:   55                      push   rbp
  4011c1:   48 89 e5                mov    rbp,rsp
  4011c4:   48 83 ec 40             sub    rsp,0x40
  4011c8:   b8 00 00 00 00          mov    eax,0x0
  4011cd:   e8 64 ff ff ff          call   401136 <return_integer>
  4011d2:   89 45 fc                mov    DWORD PTR [rbp-0x4],eax
  4011d5:   48 8d 45 d0             lea    rax,[rbp-0x30]
  4011d9:   48 89 c7                mov    rdi,rax
  4011dc:   b8 00 00 00 00          mov    eax,0x0
  4011e1:   e8 5f ff ff ff          call   401145 <return_large_struct>
  4011e6:   b8 00 00 00 00          mov    eax,0x0
  4011eb:   e8 aa ff ff ff          call   40119a <return_pair>
  4011f0:   48 89 45 c0             mov    QWORD PTR [rbp-0x40],rax
  4011f4:   48 89 55 c8             mov    QWORD PTR [rbp-0x38],rdx
  4011f8:   8b 45 fc                mov    eax,DWORD PTR [rbp-0x4]
  4011fb:   89 c6                   mov    esi,eax
  4011fd:   48 8d 05 00 0e 00 00    lea    rax,[rip+0xe00]        # 402004 <_IO_stdin_used+0x4>
  401204:   48 89 c7                mov    rdi,rax
  401207:   b8 00 00 00 00          mov    eax,0x0
  40120c:   e8 2f fe ff ff          call   401040 <printf@plt>
  401211:   48 8b 55 e8             mov    rdx,QWORD PTR [rbp-0x18]
  401215:   48 8b 45 d0             mov    rax,QWORD PTR [rbp-0x30]
  401219:   48 89 c6                mov    rsi,rax
  40121c:   48 8d 05 ee 0d 00 00    lea    rax,[rip+0xdee]        # 402011 <_IO_stdin_used+0x11>
  401223:   48 89 c7                mov    rdi,rax
  401226:   b8 00 00 00 00          mov    eax,0x0
  40122b:   e8 10 fe ff ff          call   401040 <printf@plt>
  401230:   48 8b 55 c8             mov    rdx,QWORD PTR [rbp-0x38]
  401234:   48 8b 45 c0             mov    rax,QWORD PTR [rbp-0x40]
  401238:   48 89 c6                mov    rsi,rax
  40123b:   48 8d 05 e0 0d 00 00    lea    rax,[rip+0xde0]        # 402022 <_IO_stdin_used+0x22>
  401242:   48 89 c7                mov    rdi,rax
  401245:   b8 00 00 00 00          mov    eax,0x0
  40124a:   e8 f1 fd ff ff          call   401040 <printf@plt>
  40124f:   b8 00 00 00 00          mov    eax,0x0
  401254:   c9                      leave
  401255:   c3                      ret

X86_64 From Source Code to Binary: What Does GCC Actually Do?

ddupard — Sat, 04 Jul 2026 07:03:24 +0000

When we write a C program, we work with high-level concepts like functions, variables, and loops. However, your computer's processor only understands raw binary instructions. The compiler, gcc (GNU Compiler Collection), acts as a universal translator, transforming your human-readable code into machine-executable logic.

To illustrate this, we use a simple program:

#include <stdio.h>

int main() {
    printf("Hello world\n") ;
    printf("multiple parameters %s %i %i\n","test",1,2) ;
}

1. The Invisible Compilation Journey

gcc is not just a single tool; it is an orchestrator that runs several sequential stages:

The Preprocessor: It handles directives like #include . It replaces this line with the entire content of the header file, preparing the code for compilation.

Compilation: The C code is transformed into assembly language, which is a textual representation of machine instructions.

Assembly: The assembler translates these instructions into binary format (object files .o).

Linking: This is the crucial final step. Your code uses printf, a function belonging to the standard C library (libc). The linker connects your code to the actual addresses of these functions within the system library.

2. Anatomy of a Binary: From Source to Execution

After the linking process, gcc generates an ELF File which contains several sections and several compiler-generated functions

A. Binary Sections: The Anatomy of the ELF File

.init & .fini: These sections contain code executed before and after your main function. They handle global setup and teardown.
.plt (Procedure Linkage Table): This section acts as a "trampoline" for external library functions (like printf). Since the exact memory address of libc is determined at runtime, the .plt enables dynamic linking.
.text: This is where your actual code—the main function and compiler-generated support functions—resides. It is marked as Read-Only and Executable to prevent tampering.

B. Compiler-Generated Functions

gcc adds several functions to your binary that you never explicitly wrote. These are essential for the program's lifecycle:

_start: This is the real entry point of your program. It is called by the OS kernel. It initializes the environment (arguments, environment variables) and then calls __libc_start_main.
__libc_start_main: A standard library function that prepares the environment for your main function and calls it
register_tm_clones / deregister_tm_clones: These functions handle Transactional Memory management. They are part of the GCC runtime overhead.
__do_global_dtors_aux: This function is responsible for calling destructors for global objects before the program exits.
frame_dummy: A helper function used to register frame information for exception handling.

C. Execution Flow: What Happens When You Run `./hello`?

When you launch the program from your shell, the following sequence occurs:

a. OS Kernel Loading: The Linux kernel maps the binary into memory and hands control to the dynamic linker (ld-linux.so).
b. _start: The kernel jumps to _start. It cleans up the stack, sets up the initial registers, and invokes the C library's initialization logic.
c. Initialization (.init): The code in the .init section runs, setting up the runtime environment.
d. main Invocation: __libc_start_main calls your main function.
e. printf Execution: Inside main, your code calls printf. It jumps to the .plt entry, which resolves the library address and executes the actual print command.
f. Termination: After main returns, the program invokes __do_global_dtors_aux (to clean up globals) and the .fini section, then exits by calling a kernel system call.

Summary Table

Stage	Function/Section	Responsibility
Setup	`_start`	Prepare registers and invoke `libc`
Init	`.init`	Global initializations
Logic	`main`	Your custom code
Linkage	`.plt`	Resolving library functions (`printf`)
Cleanup	`.fini`	Final program shutdown

3. The Mechanics of Parameter Passing (System V AMD64 ABI Convention)

In the C programming language, passing arguments between functions does not rely on a "magic" stack, but on a strict convention called the System V AMD64 ABI. To optimize for speed, the gcc compiler prioritizes using CPU registers before resorting to memory (the stack).

When a function is called, the parameters are placed in the following specific registers, in this exact order:

rdi: 1st argument
rsi: 2nd argument
rdx: 3rd argument
rcx: 4th argument
r8: 5th argument
r9: 6th argument

If your function requires more than 6 arguments, the additional parameters are then pushed onto the stack, which is significantly slower.

See below for some examples of the System V AMD64 ABI mechanism

In the objdump analysis of the main function, you could observe this sequence:

0000000000401156 <main>:
  401156:   f3 0f 1e fa             endbr64
  40115a:   55                      push   %rbp
  40115b:   48 89 e5                mov    %rsp,%rbp
  40115e:   48 8d 05 9f 0e 00 00    lea    0xe9f(%rip),%rax        # 402004 <_IO_stdin_used+0x4>
  401165:   48 89 c7                mov    %rax,%rdi
  401168:   e8 e3 fe ff ff          call   401050 <puts@plt>
  40116d:   b9 02 00 00 00          mov    $0x2,%ecx
  401172:   ba 01 00 00 00          mov    $0x1,%edx
  401177:   48 8d 05 92 0e 00 00    lea    0xe92(%rip),%rax        # 402010 <_IO_stdin_used+0x10>
  40117e:   48 89 c6                mov    %rax,%rsi
  401181:   48 8d 05 8d 0e 00 00    lea    0xe8d(%rip),%rax        # 402015 <_IO_stdin_used+0x15>
  401188:   48 89 c7                mov    %rax,%rdi
  40118b:   b8 00 00 00 00          mov    $0x0,%eax
  401190:   e8 cb fe ff ff          call   401060 <printf@plt>
  401195:   b8 00 00 00 00          mov    $0x0,%eax
  40119a:   5d                      pop    %rbp
  40119b:   c3                      ret

One of the most interesting aspects revealed by disassembly is GCC's optimization logic. For the simple string 'Hello world\n', the compiler identifies that no formatting is required and replaces the call to the more complex printf with a call to the more efficient puts function. This demonstrates that GCC is not just a translator, but an optimizer that actively refines your code for better performance.

For the first instruction the compiler fills the register rdi.

When calling the second printf, we can observe the compiler strictly adhering to the System V ABI. It populates the registers in the following order:

rdi: Holds the address of the format string.
rsi: Holds the address of the string argument 'test'.
rdx: Holds the first integer (1).
rcx: Holds the second integer (2).

By analyzing the assembly, we see exactly how the compiler maps your C variables to the CPU's 'highways'."

4. Epilogue

In this very short article we explained what gcc does when it compiles a C source code.
By disassembling your own code (with objdump), you precisely see what gcc creates during the compilation process. gcc adds control code (prologues/epilogues) and linking mechanisms (.plt, .got) that are essential for your program to communicate with the operating system. Moreover it follows a strict mechanism for passing arguments.

Building a comfortable workflow for debugging an old version of the Linux kernel

ddupard — Mon, 25 May 2026 01:38:52 +0000

When you want to work on the Linux kernel, for example to see how an exploit acts (like Dirty COW on kernel 4.7), you need to build a comfortable working environment.

I use:

Docker for compiling sources in their original version (with the GCC/LD versions corresponding to the source era).
QEMU for running the executables in a virtual machine.
VS Code as a debugger.

Through this article, I will show you how I configured my environment to achieve an efficient setup.

1) Compiling source code

The first step is to retrieve the kernel source code.

For relatively recent versions (beyond 5.10), the simplest way is to download the tar file from kernel.org. For older versions, the best solution is to fetch them via git.

For example, for 4.7, we start by retrieving the tags:

git ls-remote --tags git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git | grep "v4.7"

Then we clone it to get the version without downloading the entire history:

git clone --depth 1 --branch v4.7 git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 4.7

After a few minutes, you have a 4.7/ directory containing a working version of the 4.7 kernel sources.

However, these sources are not compilable as-is. Compiling a 4.7 kernel (released in 2016) with a modern compiler (GCC 13+) produces compilation errors due to changes in C language standards or security attribute handling. This is why you need to load versions of GCC and LD compatible with these sources (see the table below), and the easiest way to do this is via Docker.

Kernel Version	Ubuntu Version	Codename	Release Date	Default GCC
v2.6.x	6.06	Dapper Drake	June 2006	GCC 4.0
v3.2	12.04	Precise Pangolin	April 2012	GCC 4.6
v3.13	14.04	Trusty Tahr	April 2014	GCC 4.8
v4.4	16.04	Xenial Xerus	April 2016	GCC 5.4
v4.15	18.04	Bionic Beaver	April 2018	GCC 7.3
v5.4	20.04	Focal Fossa	April 2020	GCC 9.3
v5.15	22.04	Jammy Jellyfish	April 2022	GCC 11.2
v6.8	24.04	Noble Numbat	April 2024	GCC 13.2

We need to retrieve the Docker container corresponding to the kernel version we are interested in. For 4.7, it is Ubuntu 16.04.

To search for available containers, there are 2 solutions:
a) Run the command:

skopeo list-tags docker://docker.io/library/ubuntu

you should see something like

{
"Repository": "docker.io/library/ubuntu",
"Tags": [
"10.04",
"12.04",
"12.04.5",
"12.10",
"13.04",
"13.10",
"14.04",
"14.04.1",
...

...then run the command:

sudo docker pull ubuntu:16.04

b) Go to Docker Hub

and download it.

After a few minutes, you can run the following command to check that your container was downloaded

sudo docker images

and you should see

kernel-builder-515:latest 9f311ec05f16 580MB 147MB U

ubuntu:16.04 1f1a2d56de1d 195MB 46.5MB U

I always favor LTS (Long Term Support) versions like 16.04, because they guarantee library longevity and compilation tool stability, which is critical to avoid linking errors when generating vmlinux.

See the difference here: ubuntu:16.04 is the raw image we just downloaded, while kernel-builder-515 represents the kernel 5.15 image that I customized to include all the necessary compilation tools. We will now cover this customization step using the Dockerfile.

To customize the downloaded Docker container, we use a Dockerfile, here is an example:

### Example for kernel 4.7 (based on Ubuntu 16.04 - Xenial)
FROM ubuntu:16.04

### Install compilation dependencies
RUN apt-get update && apt-get install -y \
    build-essential \
    libncurses5-dev \
    bison \
    flex \
    libssl-dev \
    libelf-dev \
    gcc-5 g++-5 \
    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 100

WORKDIR /usr/src/linux

To create a personal container version, just run the command:

sudo docker build -t my-ubuntu:16.04 .

if we launch sudo docker images, you should see the following result

kernel-builder-515:latest 9f311ec05f16 580MB 147MB U

my-ubuntu:16.04 3aa5b81c120f 822MB 188MB U

ubuntu:16.04 1f1a2d56de1d 195MB 46.5MB U

You can find other Dockerfile examples at https://github.com/ddupard/low-level-security-research.

If you have followed all previous steps, you should be able to compile without issues by running these commands in a shell:

sudo docker run -it \
  -v ~/Desktop/KERNEL/4.7:/build \
  my-ubuntu:16.04

make x86_64_defconfig

scripts/config --enable CONFIG_DEBUG_INFO
scripts/config --disable CONFIG_DEBUG_INFO_REDUCED
scripts/config --enable CONFIG_GDB_SCRIPTS
scripts/config --enable CONFIG_DEBUG_KERNEL
scripts/config --disable CONFIG_RANDOMIZE_BASE

make olddefconfig
make scripts_gdb

make -j$(nproc)

et voila

2) Creating the runtime

This step consists of two parts:

Retrieving and compiling BusyBox.
Creating the initramfs.cpio.gz.

Unlike a complete Linux distribution that weighs gigabytes, BusyBox bundles essential utilities (ls, cat, sh, etc.) into a single binary executable.

We start by creating a shell script (build_busybox.sh) to download the BusyBox sources and compile them within Docker using my-ubuntu:16.04.

#!/bin/bash

BUSYBOX_VERSION="1.36.1"
BUSYBOX_TAR="busybox-$BUSYBOX_VERSION.tar.bz2"
BUILD_DIR="$(pwd)/busybox_build"

if [ ! -f "$BUSYBOX_TAR" ]; then
    echo "Downloading BusyBox $BUSYBOX_VERSION..."
    wget "https://busybox.net/downloads/$BUSYBOX_TAR"
fi

tar xvf "$BUSYBOX_TAR"
cd "busybox-$BUSYBOX_VERSION"

make defconfig
sed -i 's/.*CONFIG_STATIC.*/CONFIG_STATIC=y/' .config

### 4. Compilation
echo "Compilation of BusyBox..."
make -j$(nproc)
make install CONFIG_PREFIX="$BUILD_DIR"

echo "BusyBox compiled in : $BUILD_DIR"

then we launch the following command

sudo docker run -it \
  -v "$(pwd)":/build \
  -w /build \
  my-ubuntu:16.04 \
  /bin/bash ./build_busybox.sh

the compilation fails with the following message indicating that wget and other executables are missing

sudo docker run -it \
-v "$(pwd)":/build \
-w /build \
my-ubuntu:16.04 \
/bin/bash ./build_busybox.sh
Téléchargement de BusyBox 1.36.1...
./build_busybox.sh: line 11: wget: command not found
tar: busybox-1.36.1.tar.bz2: Cannot open: No such file or directory
tar: Error is not recoverable: exiting now

so we need to modify the Dockerfile in order to include these new executables

FROM ubuntu:16.04

RUN apt-get update && apt-get install -y \
    build-essential \
    gcc \
    make \
    libncurses5-dev \
    libssl-dev \
    bc \
    bison \
    flex \
    nano \
    git \
    cpio \
    openssh-client \
    wget \
    bzip2 \
    ca-certificates \
    linux-headers-generic \
    && rm -rf /var/lib/apt/lists/*


WORKDIR /opt
RUN git clone https://github.com/radareorg/radare2 \
    && radare2/sys/install.sh --install \
    && rm -rf radare2

WORKDIR /build

now we can recreate the container using

sudo docker build -t my-ubuntu:16.04 .

then we rebuild busybox with the following command

sudo docker run -it \
  -v "$(pwd)":/build \
  -w /build \
  my-ubuntu:16.04 \
  /bin/bash ./build_busybox.sh

however the compilation process wil fail with the following message

CC miscutils/mt.o
CC miscutils/nandwrite.o
CC miscutils/partprobe.o
CC miscutils/raidautorun.o
CC miscutils/readahead.o
CC miscutils/runlevel.o
CC miscutils/rx.o
CC miscutils/seedrng.o
miscutils/seedrng.c:45:24: fatal error: sys/random.h: No such file or directory

it's because you are compiling a recent version of BusyBox with an old Ubuntu 16.04 base.
in order to solve our problem, we will launch the following commands

sudo docker ps -a
in order to get the list of the last docker commands launched

CONTAINER ID	IMAGE	COMMAND	CREATED	STATUS	PORTS NAMES
49c7144e4dbb	my-ubuntu:16.04	"/bin/bash ./build_b…"	13 minutes ago	Exited (0) 10 minutes ago	mystifying_mayer
eb9d3d19f3a0	58aacbfbda6f	"/bin/bash ./build_b…"	23 minutes ago	Exited (0) 23 minutes ago	wonderful_jemison

then

sudo docker start 49c7144e4dbb
sudo docker exec -it 49c7144e4dbb /bin/bash

to restart the docker environment (root@49c7144e4dbb:/build#)
then the following commands

cd /build/busybox-1.36.1
sed -i 's/CONFIG_SEEDRNG=y/CONFIG_SEEDRNG=n/' .config

make clean
make -j$(nproc)

compilation should restart and go to the end.

then we launch the following command to make the install

make install CONFIG_PREFIX="$(pwd)/../busybox_build"

at the end a new directory should appear

Now that BusyBox has copiled, it's time to create initramfs.cpio.gz that sets up the file structure (/proc, /sys, etc.)

we are going to create and lauch a new shell script (create_initramfs.sh)

#!/bin/bash

BUILD_DIR="$(pwd)/busybox_build"
ROOTFS_DIR="$(pwd)/initramfs_staging"

rm -rf "$ROOTFS_DIR"
mkdir -p "$ROOTFS_DIR"
cp -a "$BUILD_DIR/"* "$ROOTFS_DIR/"

cd "$ROOTFS_DIR"
mkdir -p proc sys dev etc/init.d

cat <<EOF > init
#!/bin/sh
mount -t proc none /proc
mount -t sysfs none /sys
echo "--- Système démarré avec succès ---"
exec /bin/sh
EOF
chmod +x init

find . -print0 | cpio --null -ov --format=newc | gzip > ../initramfs.cpio.gz

at the end of its execution, we should see something like
./bin/mknod
./bin/more
./bin/zcat
./bin/pidof
./bin/egrep
./bin/watch
./bin/link
./bin/pwd
./bin/dd
./bin/netstat
./bin/fgrep
./bin/stty
./bin/mt
./bin/gunzip
./proc
./etc
./etc/init.d
./init
5556 blocks

and a new file in the root directory (initramfs.cpio.gz)

to check that everything is fine , we are going to launch the VM using the following command

qemu-system-x86_64 \
-kernel ./4.7/arch/x86/boot/bzImage \
-initrd ./initramfs.cpio.gz \
-append "console=ttyS0 nokaslr" \
-nographic

and something like below should appear

if it happens , congrats your environment is working now

3) Debugging setup

To debug the kernel, the base tool is GDB, but it isn't very user-friendly. The idea here is to use VS Code's overlay for GDB.

First, ensure you compile with debug options in your compile_4.7.sh script:

./scripts/config --enable CONFIG_DEBUG_INFO
./scripts/config --disable CONFIG_DEBUG_INFO_REDUCED
./scripts/config --enable CONFIG_GDB_SCRIPTS
./scripts/config --enable CONFIG_DEBUG_KERNEL
./scripts/config --disable CONFIG_RANDOMIZE_BASE

Verify with:

file ./4.7/vmlinux

(You should see a line containing "with debug_info")

Then, launch QEMU with the -s and -S options:

-s: Opens a GDB server on tcp::1234.
-S: Freezes QEMU at the first instruction.

qemu-system-x86_64 \
-kernel ./4.7/arch/x86/boot/bzImage \
-initrd ./initramfs.cpio.gz \
-append "console=ttyS0 nokaslr" \
-nographic \
-s -S

In VS Code, install the C/C++ extension from Microsoft and create a .vscode/launch.json file:

{
  "version": "0.2.0",
  "configurations": [
    {
      "name": "(gdb) Attacher au Kernel",
      "type": "cppdbg",
      "request": "launch",
      "program": "${workspaceFolder}/4.7/vmlinux",
      "miDebuggerServerAddress": "localhost:1234",
      "miDebuggerPath": "/usr/bin/gdb",
      "setupCommands": [
        { "text": "-enable-pretty-printing" }
      ],
      "cwd": "${workspaceFolder}"
    }
  ]
}

Debugging Workflow

Launch QEMU (using your script with -s -S).
In VS Code, go to the 4.7 directory and launch the "Debug Kernel LKM" configuration.
This triggers the kernel to start. You can now set breakpoints, view registers, and use the debug console to launch commands like the following

-exec info functions
-exec p/x $rax
-exec monitor xp/512gx 0x4786000

Note: If you load an LKM, you must load the symbol file and tell GDB where the source files are located using:
-exec add-symbol-file /path/to/my_attack.ko ...
-exec set substitute-path ...

Happy kernel debugging!

Appendix: Automation Script

To make the environment setup more efficient, I use a script (run_qemu.sh) to quickly switch between different kernel versions and automate the LKM injection process for the 5.15 LTS kernel.

You can save the following code into a file named run_qemu.sh in your ~/Desktop/KERNEL directory. Make sure to give it execution permissions with chmod +x run_qemu.sh.

#!/bin/bash

# This script runs on the host (Ubuntu) in ~/Desktop/KERNEL
cd "$HOME/Desktop/KERNEL"

echo "-------------------------------------------------------"
echo "  COGNITIVE FIREWALL LAB - TEST ENVIRONMENT"
echo "-------------------------------------------------------"
echo "Select the kernel version to test:"
echo "1) Kernel 4.7  (Dirty COW vulnerable)"
echo "2) Kernel 4.8  (Intro XDP)"
echo "3) Kernel 4.9  (Dirty COW patched)"
echo "4) Kernel 5.15 (LTS)"
echo "q) Quit"
echo "-------------------------------------------------------"

read -p "Your choice [1-4] : " choice

case $choice in
    1) VERSION="4.7"; KERNEL_PATH="./4.7/arch/x86/boot/bzImage" ;;
    2) VERSION="4.8"; KERNEL_PATH="./4.8/arch/x86/boot/bzImage" ;;
    3) VERSION="4.9"; KERNEL_PATH="./4.9/arch/x86/boot/bzImage" ;;        
    4)
        VERSION="5.15"
        KERNEL_PATH="./5.15/arch/x86/boot/bzImage"
        LKM2_PATH="./lkm/poc/my_attack.ko"

        if [ -f "$LKM2_PATH" ]; then
            echo "Building initramfs for Kernel 5.15..."
            mkdir -p .initramfs_root
            cd .initramfs_root
            zcat ../initramfs.cpio.gz | cpio -idmv &>/dev/null
            cp "../$LKM2_PATH" .

            # Automatic LKM injection script
            cat << 'EOF' > init
#!/bin/sh
mount -t proc proc /proc
mount -t sysfs sysfs /sys
mount -t devtmpfs devtmpfs /dev 2>/dev/null
if [ -f /my_attack.ko ]; then insmod /my_attack.ko; fi
exec /bin/sh
EOF
            chmod +x init
            find . -print0 | cpio --null -ov --format=newc 2>/dev/null | gzip -9 > ../initramfs.cpio.gz
            cd ..
            rm -rf .initramfs_root
        fi
        qemu-system-x86_64 -kernel "$KERNEL_PATH" -initrd "$HOME/Desktop/KERNEL/initramfs.cpio.gz" -nographic -append "console=ttyS0 loglevel=7 nosmep nosmap nokaslr" -s -S
        exit 0
        ;;
    q) exit 0 ;;
    *) echo "Invalid choice"; exit 1 ;;
esac

# Launch QEMU for versions 1, 2, 3
qemu-system-x86_64 \
  -kernel "$KERNEL_PATH" \
  -initrd "$HOME/Desktop/KERNEL/initramfs.cpio.gz" \
  -nographic \
  -append "console=ttyS0 nokaslr loglevel=7" \
  -s -S