lib/bcheck: Prevent libc_malloc/libc_free etc from being miscompiled
lib/bcheck: Prevent libc_malloc/libc_free etc from being miscompiled

On i386 and gcc-4.7 I found that libc_malloc was miscompiled - look:

static void *libc_malloc(size_t size)
{
    void *ptr;
    restore_malloc_hooks();     // __malloc_hook = saved_malloc_hook
    ptr = malloc(size);
    install_malloc_hooks();     // saved_malloc_hook = __malloc_hook, __malloc_hook = __bound_malloc
    return ptr;
}

	.type	libc_malloc, @function
libc_malloc:
.LFB56:
	.cfi_startproc
	pushl	%edx
	.cfi_def_cfa_offset 8
	movl	%eax, (%esp)
	call	malloc
	movl	$__bound_malloc, __malloc_hook
	movl	$__bound_free, __free_hook
	movl	$__bound_realloc, __realloc_hook
	movl	$__bound_memalign, __memalign_hook
	popl	%ecx
	.cfi_def_cfa_offset 4
	ret

Here gcc inlined both restore_malloc_hooks() and install_malloc_hooks()
and decided that

    saved_malloc_hook -> __malloc_hook -> saved_malloc_hook

stores are not needed and could be ommitted. Only it did not know
__molloc_hook affects malloc()...

So add compiler barrier to both install and restore hooks functions and
be done with it - the code is now ok:

    diff --git a/bcheck0.s b/bcheck1.s
    index 5f50293..4c02a5f 100644
    --- a/bcheck0.s
    +++ b/bcheck1.s
    @@ -42,8 +42,24 @@ libc_malloc:
            .cfi_startproc
            pushl   %edx
            .cfi_def_cfa_offset 8
    +       movl    saved_malloc_hook, %edx
    +       movl    %edx, __malloc_hook
    +       movl    saved_free_hook, %edx
    +       movl    %edx, __free_hook
    +       movl    saved_realloc_hook, %edx
    +       movl    %edx, __realloc_hook
    +       movl    saved_memalign_hook, %edx
    +       movl    %edx, __memalign_hook
            movl    %eax, (%esp)
            call    malloc
    +       movl    __malloc_hook, %edx
    +       movl    %edx, saved_malloc_hook
    +       movl    __free_hook, %edx
    +       movl    %edx, saved_free_hook
    +       movl    __realloc_hook, %edx
    +       movl    %edx, saved_realloc_hook
    +       movl    __memalign_hook, %edx
    +       movl    %edx, saved_memalign_hook
            movl    $__bound_malloc, __malloc_hook
            movl    $__bound_free, __free_hook
            movl    $__bound_realloc, __realloc_hook

For barrier I use

    __asm__ __volatile__ ("": : : "memory")

which is used as compiler barrier by Linux kernel, and mentioned in gcc
docs and in wikipedia [1].

Without this patch any program compiled with tcc -b crashes in startup
because of infinite recursion in libc_malloc.

[1] http://en.wikipedia.org/wiki/Memory_ordering#Compiler_memory_barrier

leeduhem/tinycc,Saectar/inherpreter,run4flat/tinycc,avih/tinycc,run4flat/tinycc,alfonsokim/tinycc,FriedBreakfast/tcc,fstudio/tinycc,Archs/TinyCC,andreiw/tinycc,seyko2/tinycc,code4game/tinycc,Saectar/inherpreter,mingodad/tinycc,avih/tinycc,FriedBreakfast/tcc,seyko2/tinycc,andrewchambers/tccmirror,acook/tinycc,andrewchambers/tccmirror,yousong/tinycc,alfonsokim/tinycc,seyko2/tinycc,andreiw/tinycc,FriedBreakfast/tcc,andrewchambers/tccmirror,gfwilliams/tinycc,gfwilliams/tinycc,acook/tinycc,avih/tinycc,metaeducation/tcc,mirror/tinycc,ftdi-paul-jiao/tinyCC,metaeducation/tcc,code4game/tinycc,seyko2/tinycc,code4game/tinycc,mirror/tinycc,melkior/tinycc,andrewchambers/tccmirror,Saectar/inherpreter,run4flat/tinycc,run4flat/tinycc,gfwilliams/tinycc,rchandramouli/tinycc,Saectar/inherpreter,mobiphil/tinnycc,ftdi-paul-jiao/tinyCC,acook/tinycc,fstudio/tinycc,mobiphil/tinnycc,rchandramouli/tinycc,rchandramouli/tinycc,acook/tinycc,Archs/TinyCC,gfwilliams/tinycc,yousong/tinycc,alfonsokim/tinycc,Saectar/inherpreter,Archs/TinyCC,ftdi-paul-jiao/tinyCC,mingodad/tinycc,andrewchambers/tccmirror,yousong/tinycc,rchandramouli/tinycc,mirror/tinycc,mirror/tinycc,melkior/tinycc,avih/tinycc,melkior/tinycc,Archs/TinyCC,fstudio/tinycc,ftdi-paul-jiao/tinyCC,Archs/TinyCC,fstudio/tinycc,melkior/tinycc,gfwilliams/tinycc,FriedBreakfast/tcc,seyko2/tinycc,leeduhem/tinycc,melkior/tinycc,metaeducation/tcc,FriedBreakfast/tcc,metaeducation/tcc,leeduhem/tinycc,andreiw/tinycc,alfonsokim/tinycc,rchandramouli/tinycc,mingodad/tinycc,leeduhem/tinycc,yousong/tinycc,leeduhem/tinycc,fstudio/tinycc,alfonsokim/tinycc,mobiphil/tinnycc,run4flat/tinycc,mobiphil/tinnycc,metaeducation/tcc,mingodad/tinycc