[PATCH] x86: Allow percpu variables to be page-aligned

Let's allow page-alignment in general for per-cpu data (wanted by Xen, and
Ingo suggested KVM as well).

Because larger alignments can use more room, we increase the max per-cpu
memory to 64k rather than 32k: it's getting a little tight.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Acked-by: Ingo Molnar <mingo@elte.hu>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/alpha/kernel/vmlinux.lds.S b/arch/alpha/kernel/vmlinux.lds.S
index 4cc44bd33d3345502339ebe62e45790841eddff0..cf1e6fc6c6869bc81d0e0c15d6caf522f4288e99 100644 (file)
--- a/arch/alpha/kernel/vmlinux.lds.S
+++ b/arch/alpha/kernel/vmlinux.lds.S
@@ -69,7 +69,7 @@ SECTIONS
   . = ALIGN(8);
   SECURITY_INIT
 
-  . = ALIGN(64);
+  . = ALIGN(8192);
   __per_cpu_start = .;
   .data.percpu : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index ddbdad48f5b2636285c36d4836bb863f5b2e25f7..d1a6a597ed9ad111c11d5ae6680d17ee082e2e91 100644 (file)
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -59,7 +59,7 @@ SECTIONS
                        usr/built-in.o(.init.ramfs)
                __initramfs_end = .;
 #endif
-               . = ALIGN(64);
+               . = ALIGN(4096);
                __per_cpu_start = .;
                        *(.data.percpu)
                __per_cpu_end = .;

diff --git a/arch/cris/arch-v32/vmlinux.lds.S b/arch/cris/arch-v32/vmlinux.lds.S
index e124fcd766d5c0fcd1cfd06fd42a746e5444822b..dfa25e1542b91dec2a52a5f044c45589956c702f 100644 (file)
--- a/arch/cris/arch-v32/vmlinux.lds.S
+++ b/arch/cris/arch-v32/vmlinux.lds.S
@@ -91,6 +91,7 @@ SECTIONS
        }
        SECURITY_INIT
 
+       . =  ALIGN (8192);
        __per_cpu_start = .;
        .data.percpu  : { *(.data.percpu) }
        __per_cpu_end = .;

diff --git a/arch/frv/kernel/vmlinux.lds.S b/arch/frv/kernel/vmlinux.lds.S
index 97910e016825e29361f4fb41b5e5aee69b043d60..28eae9735ad6b9f5b189699c61518c20a7dc1859 100644 (file)
--- a/arch/frv/kernel/vmlinux.lds.S
+++ b/arch/frv/kernel/vmlinux.lds.S
@@ -57,6 +57,7 @@ SECTIONS
   __alt_instructions_end = .;
  .altinstr_replacement : { *(.altinstr_replacement) }
 
+  . = ALIGN(4096);
   __per_cpu_start = .;
   .data.percpu  : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/i386/kernel/vmlinux.lds.S b/arch/i386/kernel/vmlinux.lds.S
index f4ec722318358d2675dc57c1a58e55b0e3b72041..97fe6eac47c9c581e0f23d2f175f123d4264554f 100644 (file)
--- a/arch/i386/kernel/vmlinux.lds.S
+++ b/arch/i386/kernel/vmlinux.lds.S
@@ -194,7 +194,7 @@ SECTIONS
        __initramfs_end = .;
   }
 #endif
-  . = ALIGN(L1_CACHE_BYTES);
+  . = ALIGN(4096);
   .data.percpu  : AT(ADDR(.data.percpu) - LOAD_OFFSET) {
        __per_cpu_start = .;
        *(.data.percpu)

diff --git a/arch/m32r/kernel/vmlinux.lds.S b/arch/m32r/kernel/vmlinux.lds.S
index 439cc257cd1df7faf55a57051530d65be3dac13b..6c73bca3f478cf35d88f8e4a9b169b77bae2d617 100644 (file)
--- a/arch/m32r/kernel/vmlinux.lds.S
+++ b/arch/m32r/kernel/vmlinux.lds.S
@@ -110,7 +110,7 @@ SECTIONS
   __initramfs_end = .;
 #endif
 
-  . = ALIGN(32);
+  . = ALIGN(4096);
   __per_cpu_start = .;
   .data.percpu  : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S
index c76b793310c26d4a5a5c4895a7d83fe839826165..043f637e3d106c02c8e38396eba3ea8da7e33723 100644 (file)
--- a/arch/mips/kernel/vmlinux.lds.S
+++ b/arch/mips/kernel/vmlinux.lds.S
@@ -119,7 +119,7 @@ SECTIONS
   .init.ramfs : { *(.init.ramfs) }
   __initramfs_end = .;
 #endif
-  . = ALIGN(32);
+  . = ALIGN(_PAGE_SIZE);
   __per_cpu_start = .;
   .data.percpu  : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S
index 2a8253358c6ccc6686a9a255d171c6fa3f59e229..c745859905982ad518940c73fb36f498a837ba59 100644 (file)
--- a/arch/parisc/kernel/vmlinux.lds.S
+++ b/arch/parisc/kernel/vmlinux.lds.S
@@ -181,7 +181,7 @@ SECTIONS
   .init.ramfs : { *(.init.ramfs) }
   __initramfs_end = .;
 #endif
-  . = ALIGN(32);
+  . = ALIGN(ASM_PAGE_SIZE);
   __per_cpu_start = .;
   .data.percpu  : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 22083ce3cc30c9106766afc641b8436df42c3e84..6018178708a55f76be298d405c0fd8d76c9af6b2 100644 (file)
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -582,14 +582,14 @@ void __init setup_per_cpu_areas(void)
        char *ptr;
 
        /* Copy section for each CPU (we discard the original) */
-       size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);
+       size = ALIGN(__per_cpu_end - __per_cpu_start, PAGE_SIZE);
 #ifdef CONFIG_MODULES
        if (size < PERCPU_ENOUGH_ROOM)
                size = PERCPU_ENOUGH_ROOM;
 #endif
 
        for_each_possible_cpu(i) {
-               ptr = alloc_bootmem_node(NODE_DATA(cpu_to_node(i)), size);
+               ptr = alloc_bootmem_pages_node(NODE_DATA(cpu_to_node(i)), size);
                if (!ptr)
                        panic("Cannot allocate cpu data for CPU %d\n", i);
 

diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index 7eefeb4a30e70b9837f3e50eb045444521095baf..132067313147b37d107e3402e83b938fcc915e7a 100644 (file)
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -139,11 +139,7 @@ SECTIONS
                __initramfs_end = .;
        }
 #endif
-#ifdef CONFIG_PPC32
-       . = ALIGN(32);
-#else
-       . = ALIGN(128);
-#endif
+       . = ALIGN(PAGE_SIZE);
        .data.percpu : {
                __per_cpu_start = .;
                *(.data.percpu)

diff --git a/arch/ppc/kernel/vmlinux.lds.S b/arch/ppc/kernel/vmlinux.lds.S
index a0625562a44bb7a82db233c6388b984662ec44a8..44cd128fb719a54d724a1e9fd748917ac86c81b6 100644 (file)
--- a/arch/ppc/kernel/vmlinux.lds.S
+++ b/arch/ppc/kernel/vmlinux.lds.S
@@ -130,7 +130,7 @@ SECTIONS
   __ftr_fixup : { *(__ftr_fixup) }
   __stop___ftr_fixup = .;
 
-  . = ALIGN(32);
+  . = ALIGN(4096);
   __per_cpu_start = .;
   .data.percpu  : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S
index 418f6426a949000d0ed8fd0fccfe54e4b81c9190..e9d3432aba60965f48c136decd8bc407c918c42e 100644 (file)
--- a/arch/s390/kernel/vmlinux.lds.S
+++ b/arch/s390/kernel/vmlinux.lds.S
@@ -107,7 +107,7 @@ SECTIONS
   . = ALIGN(2);
   __initramfs_end = .;
 #endif
-  . = ALIGN(256);
+  . = ALIGN(4096);
   __per_cpu_start = .;
   .data.percpu  : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/sh/kernel/vmlinux.lds.S b/arch/sh/kernel/vmlinux.lds.S
index 78a6c09875b2b6b3dc17fc18e4855a666de69668..2f606d0ce1f6a9be3201cb4510a508ffe82c8581 100644 (file)
--- a/arch/sh/kernel/vmlinux.lds.S
+++ b/arch/sh/kernel/vmlinux.lds.S
@@ -54,7 +54,7 @@ SECTIONS
   . = ALIGN(PAGE_SIZE);
   .data.page_aligned : { *(.data.page_aligned) }
 
-  . = ALIGN(L1_CACHE_BYTES);
+  . = ALIGN(PAGE_SIZE);
   __per_cpu_start = .;
   .data.percpu : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/sh64/kernel/vmlinux.lds.S b/arch/sh64/kernel/vmlinux.lds.S
index a59c5e998131e93c5649e82d21328e8ba720d493..4f9616f39830f30ae532d1e3c67cb7adf74ddf66 100644 (file)
--- a/arch/sh64/kernel/vmlinux.lds.S
+++ b/arch/sh64/kernel/vmlinux.lds.S
@@ -85,7 +85,7 @@ SECTIONS
   . = ALIGN(PAGE_SIZE);
   .data.page_aligned : C_PHYS(.data.page_aligned) { *(.data.page_aligned) }
 
-  . = ALIGN(L1_CACHE_BYTES);
+  . = ALIGN(PAGE_SIZE);
   __per_cpu_start = .;
   .data.percpu : C_PHYS(.data.percpu) { *(.data.percpu) }
   __per_cpu_end = . ;

diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S
index e5c24e0521dee34f43e0ef32c693320512f841a9..f0bb6e60e6205ea0bb53ac43dec332aea0feca77 100644 (file)
--- a/arch/sparc/kernel/vmlinux.lds.S
+++ b/arch/sparc/kernel/vmlinux.lds.S
@@ -65,7 +65,7 @@ SECTIONS
   __initramfs_end = .;
 #endif
 
-  . = ALIGN(32);
+  . = ALIGN(4096);
   __per_cpu_start = .;
   .data.percpu  : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index d4f0a70f48456dc509e44d21d6135c8890532cc1..1fac215252e403bc25af1fbc3a58fe88e22e3994 100644 (file)
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -1343,11 +1343,11 @@ void __init setup_per_cpu_areas(void)
        /* Copy section for each CPU (we discard the original) */
        goal = PERCPU_ENOUGH_ROOM;
 
-       __per_cpu_shift = 0;
-       for (size = 1UL; size < goal; size <<= 1UL)
+       __per_cpu_shift = PAGE_SHIFT;
+       for (size = PAGE_SIZE; size < goal; size <<= 1UL)
                __per_cpu_shift++;
 
-       ptr = alloc_bootmem(size * NR_CPUS);
+       ptr = alloc_bootmem_pages(size * NR_CPUS);
 
        __per_cpu_base = ptr - __per_cpu_start;
 

diff --git a/arch/x86_64/kernel/setup64.c b/arch/x86_64/kernel/setup64.c
index 53064a9a365f1f92e707e6437d5edc0d0adbc516..64379a80d76320718b07248560b82fd943f3297b 100644 (file)
--- a/arch/x86_64/kernel/setup64.c
+++ b/arch/x86_64/kernel/setup64.c
@@ -103,9 +103,9 @@ void __init setup_per_cpu_areas(void)
                if (!NODE_DATA(cpu_to_node(i))) {
                        printk("cpu with no node %d, num_online_nodes %d\n",
                               i, num_online_nodes());
-                       ptr = alloc_bootmem(size);
+                       ptr = alloc_bootmem_pages(size);
                } else { 
-                       ptr = alloc_bootmem_node(NODE_DATA(cpu_to_node(i)), size);
+                       ptr = alloc_bootmem_pages_node(NODE_DATA(cpu_to_node(i)), size);
                }
                if (!ptr)
                        panic("Cannot allocate cpu data for CPU %d\n", i);

diff --git a/arch/x86_64/kernel/vmlinux.lds.S b/arch/x86_64/kernel/vmlinux.lds.S
index 3bdeb88d28f4ffc2a8002ea588aa339973fbe4fa..7ef0b8820cd2fcf364e3df0b08278d4f6992af40 100644 (file)
--- a/arch/x86_64/kernel/vmlinux.lds.S
+++ b/arch/x86_64/kernel/vmlinux.lds.S
@@ -195,7 +195,7 @@ SECTIONS
   __initramfs_end = .;
 #endif
 
-    . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
+  . = ALIGN(4096);
   __per_cpu_start = .;
   .data.percpu  : AT(ADDR(.data.percpu) - LOAD_OFFSET) { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/arch/xtensa/kernel/vmlinux.lds.S b/arch/xtensa/kernel/vmlinux.lds.S
index ab6370054cee709cdfccff4850d6fcdefb24ce9d..4fbd66a52a8818002bbf7165427707ec78dda339 100644 (file)
--- a/arch/xtensa/kernel/vmlinux.lds.S
+++ b/arch/xtensa/kernel/vmlinux.lds.S
@@ -198,7 +198,7 @@ SECTIONS
   __ftr_fixup : { *(__ftr_fixup) }
   __stop___ftr_fixup = .;
 
-  . = ALIGN(32);
+  . = ALIGN(4096);
   __per_cpu_start = .;
   .data.percpu  : { *(.data.percpu) }
   __per_cpu_end = .;

diff --git a/init/main.c b/init/main.c
index a92989e7836af6acb73bb6ce8b95ad76ee727f53..80f09f31bfabf68c3d6b317c7f0470f14f8bfac7 100644 (file)
--- a/init/main.c
+++ b/init/main.c
@@ -369,12 +369,8 @@ static void __init setup_per_cpu_areas(void)
        unsigned long nr_possible_cpus = num_possible_cpus();
 
        /* Copy section for each CPU (we discard the original) */
-       size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);
-#ifdef CONFIG_MODULES
-       if (size < PERCPU_ENOUGH_ROOM)
-               size = PERCPU_ENOUGH_ROOM;
-#endif
-       ptr = alloc_bootmem(size * nr_possible_cpus);
+       size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE);
+       ptr = alloc_bootmem_pages(size * nr_possible_cpus);
 
        for_each_possible_cpu(i) {
                __per_cpu_offset[i] = ptr - __per_cpu_start;

diff --git a/kernel/module.c b/kernel/module.c
index cf49ca25fcce941ae0c81e3b179b7b33654b292d..4dc4a257545c6c2500578dd046d666d796653b3f 100644 (file)
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -346,10 +346,10 @@ static void *percpu_modalloc(unsigned long size, unsigned long align,
        unsigned int i;
        void *ptr;
 
-       if (align > SMP_CACHE_BYTES) {
-               printk(KERN_WARNING "%s: per-cpu alignment %li > %i\n",
-                      name, align, SMP_CACHE_BYTES);
-               align = SMP_CACHE_BYTES;
+       if (align > PAGE_SIZE) {
+               printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",
+                      name, align, PAGE_SIZE);
+               align = PAGE_SIZE;
        }
 
        ptr = __per_cpu_start;