--- linux-2.6.7-gentoo-r19/arch/ia64/ia32/binfmt_elf32.c 2004-12-02 23:32:15.424906248 +0000 +++ linux-2.6.7-gentoo-r19/arch/ia64/ia32/binfmt_elf32.c 2004-12-02 23:35:26.813810712 +0000 @@ -82,7 +82,11 @@ ia64_elf32_init (struct pt_regs *regs) vma->vm_ops = &ia32_shared_page_vm_ops; down_write(¤t->mm->mmap_sem); { - insert_vm_struct(current->mm, vma); + if (insert_vm_struct(current->mm, vma)) { + kmem_cache_free(vm_area_cachep, vma); + up_write(¤t->mm->mmap_sem); + return; + } } up_write(¤t->mm->mmap_sem); } @@ -101,7 +105,11 @@ ia64_elf32_init (struct pt_regs *regs) vma->vm_flags = VM_READ|VM_WRITE|VM_MAYREAD|VM_MAYWRITE; down_write(¤t->mm->mmap_sem); { - insert_vm_struct(current->mm, vma); + if (insert_vm_struct(current->mm, vma)) { + kmem_cache_free(vm_area_cachep, vma); + up_write(¤t->mm->mmap_sem); + return; + } } up_write(¤t->mm->mmap_sem); } @@ -149,7 +157,7 @@ ia32_setup_arg_pages (struct linux_binpr unsigned long stack_base; struct vm_area_struct *mpnt; struct mm_struct *mm = current->mm; - int i; + int i, ret; stack_base = IA32_STACK_TOP - MAX_ARG_PAGES*PAGE_SIZE; mm->arg_start = bprm->p + stack_base; @@ -182,8 +190,12 @@ ia32_setup_arg_pages (struct linux_binpr else mpnt->vm_flags = VM_STACK_FLAGS; mpnt->vm_page_prot = (mpnt->vm_flags & VM_EXEC)? - PAGE_COPY_EXEC: PAGE_COPY; - insert_vm_struct(current->mm, mpnt); + PAGE_COPY_EXEC: PAGE_COPY; + if ((ret = insert_vm_struct(current->mm, mpnt))) { + up_write(¤t->mm->mmap_sem); + kmem_cache_free(vm_area_cachep, mpnt); + return ret; + } current->mm->total_vm = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT; } --- linux-2.6.7-gentoo-r19/arch/ia64/mm/init.c 2004-12-02 23:32:15.425906096 +0000 +++ linux-2.6.7-gentoo-r19/arch/ia64/mm/init.c 2004-12-02 23:36:46.937630040 +0000 @@ -129,7 +129,13 @@ ia64_init_addr_space (void) vma->vm_end = vma->vm_start + PAGE_SIZE; vma->vm_page_prot = protection_map[VM_DATA_DEFAULT_FLAGS & 0x7]; vma->vm_flags = VM_READ|VM_WRITE|VM_MAYREAD|VM_MAYWRITE|VM_GROWSUP; - insert_vm_struct(current->mm, vma); + down_write(¤t->mm->mmap_sem); + if (insert_vm_struct(current->mm, vma)) { + up_write(¤t->mm->mmap_sem); + kmem_cache_free(vm_area_cachep, vma); + return; + } + up_write(¤t->mm->mmap_sem); } /* map NaT-page at address zero to speed up speculative dereferencing of NULL: */ @@ -141,7 +147,13 @@ ia64_init_addr_space (void) vma->vm_end = PAGE_SIZE; vma->vm_page_prot = __pgprot(pgprot_val(PAGE_READONLY) | _PAGE_MA_NAT); vma->vm_flags = VM_READ | VM_MAYREAD | VM_IO | VM_RESERVED; - insert_vm_struct(current->mm, vma); + down_write(¤t->mm->mmap_sem); + if (insert_vm_struct(current->mm, vma)) { + up_write(¤t->mm->mmap_sem); + kmem_cache_free(vm_area_cachep, vma); + return; + } + up_write(¤t->mm->mmap_sem); } } } --- linux-2.6.7-gentoo-r19/arch/s390/kernel/compat_exec.c 2004-12-02 23:32:15.426905944 +0000 +++ linux-2.6.7-gentoo-r19/arch/s390/kernel/compat_exec.c 2004-12-02 23:39:18.846536376 +0000 @@ -39,7 +39,7 @@ int setup_arg_pages32(struct linux_binpr unsigned long stack_base; struct vm_area_struct *mpnt; struct mm_struct *mm = current->mm; - int i; + int i, ret; stack_base = STACK_TOP - MAX_ARG_PAGES*PAGE_SIZE; mm->arg_start = bprm->p + stack_base; @@ -68,7 +68,11 @@ int setup_arg_pages32(struct linux_binpr /* executable stack setting would be applied here */ mpnt->vm_page_prot = PAGE_COPY; mpnt->vm_flags = VM_STACK_FLAGS; - insert_vm_struct(mm, mpnt); + if ((ret = insert_vm_struct(mm, mpnt))) { + up_write(&mm->mmap_sem); + kmem_cache_free(vm_area_cachep, mpnt); + return ret; + } mm->total_vm = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT; } --- linux-2.6.7-gentoo-r19/arch/x86_64/ia32/ia32_binfmt.c 2004-12-02 23:32:15.427905792 +0000 +++ linux-2.6.7-gentoo-r19/arch/x86_64/ia32/ia32_binfmt.c 2004-12-02 23:41:30.438531352 +0000 @@ -330,7 +330,7 @@ int setup_arg_pages(struct linux_binprm unsigned long stack_base; struct vm_area_struct *mpnt; struct mm_struct *mm = current->mm; - int i; + int i, ret; stack_base = IA32_STACK_TOP - MAX_ARG_PAGES * PAGE_SIZE; mm->arg_start = bprm->p + stack_base; @@ -364,7 +364,11 @@ int setup_arg_pages(struct linux_binprm mpnt->vm_flags = vm_stack_flags32; mpnt->vm_page_prot = (mpnt->vm_flags & VM_EXEC) ? PAGE_COPY_EXEC : PAGE_COPY; - insert_vm_struct(mm, mpnt); + if ((ret = insert_vm_struct(mm, mpnt))) { + up_write(&mm->mmap_sem); + kmem_cache_free(vm_area_cachep, mpnt); + return ret; + } mm->total_vm = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT; } --- linux-2.6.7-gentoo-r19/fs/exec.c 2004-12-02 23:32:15.428905640 +0000 +++ linux-2.6.7-gentoo-r19/fs/exec.c 2004-12-02 23:33:06.941074600 +0000 @@ -342,7 +342,7 @@ int setup_arg_pages(struct linux_binprm unsigned long stack_base; struct vm_area_struct *mpnt; struct mm_struct *mm = current->mm; - int i; + int i, ret; long arg_size; #ifdef CONFIG_STACK_GROWSUP @@ -413,7 +413,6 @@ int setup_arg_pages(struct linux_binprm down_write(&mm->mmap_sem); { - struct vm_area_struct *vma; mpnt->vm_mm = mm; #ifdef CONFIG_STACK_GROWSUP mpnt->vm_start = stack_base; --- linux-2.6.7-gentoo-r19/include/linux/mm.h 2004-12-02 23:32:15.430905336 +0000 +++ linux-2.6.7-gentoo-r19/include/linux/mm.h 2004-12-02 23:33:06.942074448 +0000 @@ -623,7 +623,7 @@ extern struct vm_area_struct *vma_merge( extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); extern int split_vma(struct mm_struct *, struct vm_area_struct *, unsigned long addr, int new_below); -extern void insert_vm_struct(struct mm_struct *, struct vm_area_struct *); +extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *, struct rb_node **, struct rb_node *); extern struct vm_area_struct *copy_vma(struct vm_area_struct **, --- linux-2.6.7-gentoo-r19/mm/mmap.c 2004-12-02 23:32:15.432905032 +0000 +++ linux-2.6.7-gentoo-r19/mm/mmap.c 2004-12-02 23:33:06.944074144 +0000 @@ -1722,7 +1722,7 @@ void exit_mmap(struct mm_struct *mm) * and into the inode's i_mmap tree. If vm_file is non-NULL * then i_mmap_lock is taken here. */ -void insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma) +int insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma) { struct vm_area_struct * __vma, * prev; struct rb_node ** rb_link, * rb_parent; @@ -1745,8 +1745,9 @@ void insert_vm_struct(struct mm_struct * } __vma = find_vma_prepare(mm,vma->vm_start,&prev,&rb_link,&rb_parent); if (__vma && __vma->vm_start < vma->vm_end) - BUG(); + return -ENOMEM; vma_link(mm, vma, prev, rb_link, rb_parent); + return 0; } /*