From e19b6b72297963b429c5ac5fcbd55016144d85d8 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 22:21:10 +0000 Subject: [PATCH] multikernel: simplify and harden duplex IPI transport Replace the owner-and-slot-state producer protocol with one spinlocked producer per duplex direction and one release/acquire ready flag per slot. Reset a parent/child link only after the previous child is confirmed parked, and close endpoints before their backing image pages are released. Carry the parent and doorbell identities in the shared link, restore them through the boot device tree design, and route nested console output through the actual parent instance. Fail incompatible boot contexts and legacy image notes with explicit errors while keeping the vmlinux descriptor as the original u64 entry. Preserve host timer calibration for spawned x86 kernels and declare the OF dependency required by the upstream boot-tree manifest. Signed-off-by: Nikolay Nikolaev --- arch/x86/boot/header.S | 8 +- arch/x86/include/asm/multikernel.h | 5 +- arch/x86/include/uapi/asm/bootparam.h | 1 + arch/x86/kernel/kexec-bzimage64.c | 24 ++ arch/x86/kernel/kexec-vmlinux.c | 66 +++- arch/x86/kernel/platform-quirks.c | 4 +- arch/x86/multikernel/head_64.S | 3 +- arch/x86/multikernel/spawn.c | 133 +++++-- drivers/tty/mktty.c | 39 ++- include/linux/multikernel.h | 101 ++++-- include/linux/multikernel_abi.h | 9 + kernel/kexec_core.c | 65 ++-- kernel/multikernel/Kconfig | 1 + kernel/multikernel/core.c | 121 +++++-- kernel/multikernel/cpuset.c | 271 ++++++++++++--- kernel/multikernel/hotplug.c | 14 + kernel/multikernel/instance_dt.c | 129 +++++-- kernel/multikernel/internal.h | 9 + kernel/multikernel/ipi.c | 482 ++++++++++++-------------- kernel/multikernel/manifest.c | 14 +- 20 files changed, 1021 insertions(+), 478 deletions(-) create mode 100644 include/linux/multikernel_abi.h diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..6758247c93fd7e 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF_MK XLF_MULTIKERNEL_IPI +#else +# define XLF_MK 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF_MK cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 6f55eab4b35256..953fe71fc044a7 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -127,7 +127,7 @@ struct mk_spawn_context { u32 target_apic_id; /* Target CPU's APIC ID */ u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ - u32 reserved; /* Padding for alignment */ + u32 abi_magic; /* Host/spawn generation marker */ /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ /* Optional boot data belongs after boot_params, in the zeroed tail. */ @@ -198,7 +198,8 @@ int mk_spawn_cpu(struct mk_instance *instance, int cpu, /* The pool park set (park page, slot, page tables) as base,size pairs */ int mk_pool_park_regions(u64 *pairs, int max); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..7099f7cd167dce 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,7 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI 0x0100 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 1002fbcf2f5d0b..1341501952d81b 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -554,6 +554,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; @@ -747,10 +752,29 @@ static void *bzImage64_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { +#ifdef CONFIG_MULTIKERNEL image->arch.mk_boot_params = bootparam_load_addr; + + /* + * setup_boot_parameters() copies the host subarchitecture. A + * spawn kernel must take the multikernel platform path instead. + */ + params->hdr.hardware_subarch = X86_SUBARCH_MULTIKERNEL; + + /* + * The spawn trampoline enters the compressed kernel directly, + * bypassing purgatory. The x86 boot protocol's 64-bit entry is + * 0x200 bytes from the start of the protected-mode payload. + */ + image->arch.mk_kernel_entry = kernel_load_addr + 0x200; + ret = mk_e820_fill(image->mk_instance, params); if (ret) goto out_free_params; +#else + ret = -EOPNOTSUPP; + goto out_free_params; +#endif } /* Allocate loader specific data */ diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index a7d31a722f33b7..ef5fc7ce6941b1 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -62,12 +62,16 @@ struct elf_kernel_info { /* * Find multikernel entry point from PT_NOTE section. - * Looks for note with name "Linux" and type 0x4d4b ('MK'). + * The note type carries the generation; the descriptor remains one u64. */ -static unsigned long find_multikernel_entry_note(const void *buf, size_t len, - const Elf64_Ehdr *ehdr) +#define MK_VMLINUX_LEGACY_NOTE_TYPE 0x4d4b + +static int find_multikernel_entry_note(const void *buf, size_t len, + const Elf64_Ehdr *ehdr, + unsigned long *entry) { const Elf64_Phdr *phdrs = buf + ehdr->e_phoff; + bool legacy = false; int i; for (i = 0; i < ehdr->e_phnum; i++) { @@ -91,25 +95,34 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (ptr + note_size > end) break; - if (nhdr->n_type == 0x4d4b && - nhdr->n_namesz == 6 && + if (nhdr->n_namesz == 6 && nhdr->n_descsz == sizeof(u64) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const u64 *note_entry; + + note_entry = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + if (nhdr->n_type == MK_VMLINUX_NOTE_TYPE) { + *entry = *note_entry; + pr_info("multikernel: entry=0x%llx\n", + *note_entry); + return 0; + } + if (nhdr->n_type == MK_VMLINUX_LEGACY_NOTE_TYPE) + legacy = true; } ptr += note_size; } } - return 0; + return legacy ? -EPROTONOSUPPORT : -ENOENT; } /* * Parse ELF kernel and extract key information */ -static int kexec_parse_elf_kernel(const void *kernel_buf, unsigned long kernel_len, +static int kexec_parse_elf_kernel(const void *kernel_buf, + unsigned long kernel_len, + bool multikernel, struct elf_kernel_info *info) { const Elf64_Ehdr *ehdr; @@ -159,14 +172,25 @@ static int kexec_parse_elf_kernel(const void *kernel_buf, unsigned long kernel_l * PT_NOTE contains physical offset from load base, not virtual address. * This is the canonical way and survives symbol stripping. */ - info->multikernel_entry = find_multikernel_entry_note(kernel_buf, kernel_len, ehdr); - if (!info->multikernel_entry) { - pr_err("multikernel_startup_64 entry offset not found in PT_NOTE\n"); - return -ENOEXEC; + info->multikernel_entry = 0; + if (multikernel) { + int ret; + + ret = find_multikernel_entry_note(kernel_buf, kernel_len, ehdr, + &info->multikernel_entry); + if (ret == -EPROTONOSUPPORT) + pr_err("legacy vmlinux note type 0x%x is incompatible; expected 0x%x\n", + MK_VMLINUX_LEGACY_NOTE_TYPE, MK_VMLINUX_NOTE_TYPE); + else if (ret) + pr_err("multikernel ABI note type 0x%x not found\n", + MK_VMLINUX_NOTE_TYPE); + if (ret) + return ret == -ENOENT ? -ENOEXEC : ret; + + pr_info("Multikernel entry offset: 0x%lx\n", + info->multikernel_entry); } - pr_info("Multikernel entry offset: 0x%lx\n", info->multikernel_entry); - /* Find lowest load address and calculate total memory needed */ phdr = (const Elf64_Phdr *)(kernel_buf + ehdr->e_phoff); for (i = 0; i < ehdr->e_phnum; i++) { @@ -326,12 +350,13 @@ static void *vmlinux_load(struct kimage *image, char *kernel, .top_down = true }; struct kexec_buf pbuf = { .image = image, .buf_min = MIN_PURGATORY_ADDR, .buf_max = ULONG_MAX, .top_down = true }; + bool multikernel = image->type == KEXEC_TYPE_MULTIKERNEL; int ret; pr_info("Loading ELF vmlinux (type=%d)\n", image->type); /* Parse ELF headers */ - ret = kexec_parse_elf_kernel(kernel, kernel_len, &elf_info); + ret = kexec_parse_elf_kernel(kernel, kernel_len, multikernel, &elf_info); if (ret) { pr_err("Failed to parse ELF kernel: %d\n", ret); return ERR_PTR(ret); @@ -531,12 +556,17 @@ static void *vmlinux_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { +#ifdef CONFIG_MULTIKERNEL ret = mk_e820_fill(image->mk_instance, params); if (ret) { kvfree(ldata->kernel_buf); kfree(ldata); goto out_free_params; } +#else + ret = -EOPNOTSUPP; + goto out_free_params; +#endif } ldata->bootparams_buf = params; diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 5d4705d407b4c4..fcd674b933102c 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -47,9 +47,9 @@ static void __init multikernel_setup_calibration(void) { phys_addr_t ctx_phys = orig_boot_params - offsetof(struct mk_spawn_context, bp); - struct mk_spawn_context *ctx = __va(ctx_phys); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); - if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + if (!ctx || !ctx->boot_tsc_khz) return; multikernel_tsc_khz = ctx->boot_tsc_khz; diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..54890d936e615e 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -269,7 +270,7 @@ SYM_CODE_END(multikernel_secondary_startup) .balign 4 .long 2f - 1f .long 4f - 3f - .long 0x4d4b + .long MK_VMLINUX_NOTE_TYPE 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index c8741c95c1529e..78d9122761ff5b 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -72,6 +72,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * This kernel's own context: where its CPUs park and wake. A spawn's @@ -86,6 +87,19 @@ static struct mk_spawn_context *mk_own_context(void) return mk_pool ? mk_pool->arch.slot : NULL; } +static void *spawn_trampoline_va; +static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} + extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; extern char mk_secondary_trampoline[]; @@ -455,6 +469,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, instance->arch.spawn_ctx->boot_tsc_khz = tsc_khz; instance->arch.spawn_ctx->boot_apic_hz = (unsigned long)lapic_timer_period * HZ; + instance->arch.spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->arch.spawn_ctx); } @@ -677,18 +692,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u32 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; + } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -705,14 +740,28 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; /* * A spawn kernel cannot calibrate against legacy timers because they * belong to the host. Reuse the selected physical CPU's delay and local @@ -748,19 +797,27 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. The copy matters: * the page arrives holding the HOST kernel's trampoline code, and when * the two binaries differ its offsets are wrong for this kernel. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_own_context(); unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -768,32 +825,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - memcpy(__va(ctx->trampoline_phys), multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)__va(ctx->trampoline_phys) & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index 4a4d3f68eaf011..535c6c1fe8a3e7 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -337,6 +337,17 @@ static struct tty_driver *mktty_spawn_driver; static struct mktty_spawn_state mktty_spawn; static struct mk_ipi_handler *mktty_spawn_handler; static struct console mktty_spawn_console; + +static int mktty_spawn_parent_id(void) +{ + if (!mk_self) + return -ENODEV; + if (mk_self->id == 0) + return 0; + if (!host_instance) + return -ENODEV; + return READ_ONCE(host_instance->id); +} static bool mktty_console_registered; static int mktty_spawn_activate(struct tty_port *port, struct tty_struct *tty) @@ -376,11 +387,14 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, size_t count) { struct mktty_message *msg; - size_t sent = 0, chunk; - int ret; + size_t sent = 0, chunk, msg_size; + int ret, parent_id; if (tty->index != 0) return -ENODEV; + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + return parent_id; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) @@ -393,10 +407,10 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, msg->len = chunk; msg->reserved = 0; memcpy(msg->data, buf + sent, chunk); + msg_size = sizeof(*msg) - MKTTY_MAX_DATA + chunk; - ret = multikernel_send_ipi_data(0, msg, - sizeof(*msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + ret = multikernel_send_ipi_data(parent_id, msg, msg_size, + MKTTY_IPI_TYPE); if (ret < 0) { kfree(msg); return sent > 0 ? sent : ret; @@ -445,12 +459,16 @@ static struct mktty_message mktty_console_msg; static DEFINE_SPINLOCK(mktty_console_lock); static void mktty_spawn_console_write(struct console *con, const char *s, - unsigned int count) + unsigned int count) { unsigned long flags; - size_t chunk; + size_t chunk, msg_size; + int parent_id; spin_lock_irqsave(&mktty_console_lock, flags); + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + goto out; while (count > 0) { chunk = min_t(size_t, count, MKTTY_MAX_DATA); mktty_console_msg.type = MKTTY_MSG_OUTPUT; @@ -458,13 +476,14 @@ static void mktty_spawn_console_write(struct console *con, const char *s, mktty_console_msg.len = chunk; mktty_console_msg.reserved = 0; memcpy(mktty_console_msg.data, s, chunk); + msg_size = sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk; - multikernel_send_ipi_data(0, &mktty_console_msg, - sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + (void)multikernel_send_ipi_data_to_host(&mktty_console_msg, + msg_size, MKTTY_IPI_TYPE); s += chunk; count -= chunk; } +out: spin_unlock_irqrestore(&mktty_console_lock, flags); } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 1ec6a0d5170005..09460a911bb9cc 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -16,6 +16,11 @@ #include #include #include +#include +#include + +struct pci_bus; +struct mk_instance; #ifdef CONFIG_MULTIKERNEL #include @@ -40,6 +45,7 @@ typedef u64 mk_phys_cpu_t; #define MK_PHYS_CPU_INVALID (~(mk_phys_cpu_t)0) struct mk_cpu_set { + raw_spinlock_t lock; unsigned int nr; /* Entries in use */ unsigned int cap; /* Allocated capacity */ mk_phys_cpu_t *ids; @@ -55,25 +61,15 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id); int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src); int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set); -static inline unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) -{ - return set ? set->nr : 0; -} - -static inline bool mk_cpu_set_empty(const struct mk_cpu_set *set) -{ - return mk_cpu_set_count(set) == 0; -} - -static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) -{ - return mk_cpu_set_empty(set) ? MK_PHYS_CPU_INVALID : set->ids[0]; -} +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set); +bool mk_cpu_set_empty(const struct mk_cpu_set *set); +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set); +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id); #define mk_cpu_set_for_each(i, id, set) \ for ((i) = 0; \ - (set) && (i) < (set)->nr && \ - (((id) = (set)->ids[(i)]), true); \ + mk_cpu_set_get((set), (i), &(id)); \ (i)++) /** @@ -88,16 +84,15 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + u32 ready; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ - unsigned int type; /* User-defined type identifier */ + unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ char buffer[MK_MAX_DATA_SIZE]; /* Actual data buffer */ }; /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ }; @@ -109,9 +104,10 @@ struct mk_ipi_ring { /* Presence table capacity, in CPUs (see mk_cpu_rank) */ #define MK_PARKED_MAX 512 -/* Shared memory structures - per-instance design */ +/* One duplex link per parent/child pair. Each ring has one kernel producer. */ struct mk_shared_data { - struct mk_ipi_ring ring; /* IPI message ring buffer */ + struct mk_ipi_ring to_child; + struct mk_ipi_ring to_parent; /* * Force-halt marker, host-owned. Armed before the host NMIs the * instance's CPUs and cleared with the rest of this struct when @@ -131,6 +127,40 @@ struct mk_shared_data { * the struct on re-exec. */ u8 parked[MK_PARKED_MAX]; + s32 parent_id; + s32 child_id; + u32 reserved; + u64 parent_doorbell_cpu; + u64 child_doorbell_cpu; +}; + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + unsigned int i; + + for (i = 0; i < MK_IPI_RING_SIZE; i++) + WRITE_ONCE(ring->entries[i].ready, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->to_child); + mk_ipi_ring_reset(&shared->to_parent); + WRITE_ONCE(shared->force_halt, 0); +} + +struct mk_ipi_endpoint { + struct mk_ipi_ring *tx; + struct mk_ipi_ring *rx; + raw_spinlock_t tx_lock; + raw_spinlock_t rx_lock; + u32 tx_head; + u32 rx_tail; + bool tx_enabled; + bool rx_dispatching; + bool parent_side; + bool registered; + struct list_head rx_node; }; /* Function pointer type for IPI callbacks */ @@ -172,11 +202,18 @@ void multikernel_unregister_handler(struct mk_ipi_handler *handler); * Returns 0 on success, negative error code on failure */ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type); +int multikernel_send_ipi_data_to_host(void *data, size_t data_size, + unsigned long type); void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side); +void mk_ipi_endpoint_unregister(struct mk_instance *instance); +void mk_ipi_endpoint_close(struct mk_instance *instance); +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu); +void mk_ipi_handlers_enable(void); /* * Multikernel Messaging System @@ -655,6 +692,11 @@ struct mk_instance { struct mk_shared_data *ipi_data; /* IPI shared memory buffer (virtual address) */ phys_addr_t ipi_phys; /* IPI buffer physical address */ u32 ipi_pages; /* IPI buffer size in pages */ + /* + * Separate host self area used when a spawn fences its parent. + * NULL for ordinary child records, whose halt area is @ipi_data. + */ + struct mk_shared_data *halt_data; /* * On a spawn's record of its host: the physical address of the * host's pool wake slot, where the host's CPUs park. A backup @@ -669,6 +711,7 @@ struct mk_instance { * explicitly so routing never masquerades as ownership in @cpus. */ mk_phys_cpu_t ipi_target; + struct mk_ipi_endpoint ipi_endpoint; /* Kexec integration */ struct kimage *kimage; /* Associated kimage object */ @@ -922,6 +965,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); @@ -936,6 +980,7 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); @@ -991,6 +1036,11 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -998,6 +1048,7 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 +/* Bumped whenever the shared-memory layout or message semantics change. */ #define MK_FDT_COMPATIBLE "multikernel-v1" /** @@ -1015,8 +1066,6 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) * the host side, and restore an instance from it on the spawn side. */ - - /** * mk_instance_restore_from_manifest() - Restore this instance from the manifest * @@ -1099,6 +1148,8 @@ void mk_arch_register_cpu(mk_phys_cpu_t phys_id); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86, SDEI or diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..3d9bb5fdeee61e --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* One generation marker shared by the spawn context and image capabilities. */ +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b0002 +#define MK_VMLINUX_NOTE_TYPE 0x4d4b0002 + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index e1d0d55b1a0f46..709838b7dfe1da 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -609,6 +609,12 @@ void kimage_free(struct kimage *image) if (image->type == KEXEC_TYPE_MULTIKERNEL) { unsigned long i; + /* Stop delivery before image-owned shared pages are returned. */ +#ifdef CONFIG_MULTIKERNEL + if (image->mk_instance) + mk_ipi_endpoint_unregister(image->mk_instance); +#endif + for (i = 0; i < image->nr_segments; i++) { void *virt_addr = phys_to_virt(image->segment[i].mem); @@ -1701,6 +1707,12 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1755,10 +1767,11 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1775,32 +1788,36 @@ int multikernel_kexec_by_id(int mk_id) PAGE_ALIGN(sizeof(struct mk_shared_data)) >> PAGE_SHIFT; } - /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. - */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); + /* Reset only this parent/child link, after the old child is parked. */ + if (instance->ipi_data) { + mk_ipi_link_reset(instance, mk_self->id, mk_id, + mk_cpu_set_first(mk_self->cpus), + mk_cpu_set_first(instance->cpus)); + } + rc = mk_arch_spawn_instance(mk_image, instance, cpu); + if (rc) { + mk_ipi_endpoint_close(instance); + goto unlock; + } /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * The instance is running once its CPUs leave the park loop. Publish that + * state before dropping the global kexec lock so another exec cannot race + * this boot while the readiness handshake is pending. */ - mk_ipi_ring_drop_pending(); + rc = mk_instance_set_kexec_active(mk_image->mk_id); + if (rc) { + int abort_ret = mk_instance_abort_spawn(instance); - rc = mk_arch_spawn_instance(mk_image, instance, cpu); - if (rc == 0) { - rc = mk_instance_set_kexec_active(mk_image->mk_id); - if (rc) - pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); + if (abort_ret) + pr_crit("Instance %d activation abort failed: %d\n", + mk_id, abort_ret); + goto unlock; } + kexec_unlock(); + return 0; + unlock: kexec_unlock(); return rc; diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index cec1ea32d83361..e7e242d0feac37 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -17,6 +17,7 @@ config MULTIKERNEL depends on KEXEC_CORE depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE + depends on OF select LIBFDT select OF_DYNAMIC if OF help diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 9de50f787d2737..cecdebdcc36d6c 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -171,6 +171,7 @@ static void mk_instance_release(struct kref *kref) pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); + mk_ipi_endpoint_unregister(instance); mk_instance_return_all_cpus(instance); mk_instance_return_pci_devices(instance); @@ -178,6 +179,8 @@ static void mk_instance_release(struct kref *kref) mk_instance_free_memory(instance); mk_instance_track_dump(instance, instance->state, MK_STATE_READY); + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -231,6 +234,9 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) instance->state = MK_STATE_READY; instance->ipi_target = MK_PHYS_CPU_INVALID; + raw_spin_lock_init(&instance->ipi_endpoint.tx_lock); + raw_spin_lock_init(&instance->ipi_endpoint.rx_lock); + INIT_LIST_HEAD(&instance->ipi_endpoint.rx_node); INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); @@ -297,6 +303,8 @@ void mk_instance_free(struct mk_instance *instance) list_del(&plat_dev->list); kfree(plat_dev); } + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -423,12 +431,25 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; /* Empty until the instance first ran, so nothing of it is executing */ - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + if (!instance->cpus_on_slot) + return 0; + + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } + + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -436,6 +457,7 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -1542,7 +1564,12 @@ static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) */ void __noreturn mk_halt_to_pool(void) { - mk_notify_down_and_park(0, MK_SYS_HALTED); + int parent_id; + + if (!host_instance) + panic("multikernel: spawned kernel has no parent instance"); + parent_id = READ_ONCE(host_instance->id); + mk_notify_down_and_park(parent_id, MK_SYS_HALTED); } static void mk_shutdown_work_fn(struct work_struct *work) @@ -1717,7 +1744,7 @@ int multikernel_halt_by_id(int mk_id) return ret; } -/** +/* * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI * @mk_id: Instance ID to halt * @@ -1788,7 +1815,7 @@ EXPORT_SYMBOL_GPL(mk_cpu_rank); static int mk_fence_missing(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing = 0; @@ -1813,7 +1840,7 @@ static int mk_fence_missing(struct mk_instance *instance, static int mk_confirm_fenced(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing, ret; @@ -1847,22 +1874,20 @@ static int mk_confirm_fenced(struct mk_instance *instance, return ret; } -int multikernel_force_halt_by_id(int mk_id) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { - struct mk_instance *instance; struct mk_cpu_set *targets; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; + return -EINVAL; if (instance == mk_self) { - pr_err("Cannot force halt this kernel (id %d)\n", mk_id); - mk_instance_put(instance); + pr_err("Cannot force halt this kernel (id %d)\n", instance->id); return -EINVAL; } @@ -1873,29 +1898,27 @@ int multikernel_force_halt_by_id(int mk_id) * a rerun the instance is stuck for good. The parent is ACTIVE. */ if (instance->state != MK_STATE_ACTIVE && - instance->state != MK_STATE_LOADED) { + (!allow_loaded || instance->state != MK_STATE_LOADED)) { pr_err("Instance %d not running (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } targets = mk_cpu_set_alloc(); - if (!targets) { - mk_instance_put(instance); + if (!targets) return -ENOMEM; - } ret = mk_force_halt_targets(instance, targets); if (!ret && mk_cpu_set_empty(targets)) ret = -EINVAL; if (ret) { - pr_err("Instance %d: no force-halt targets: %d\n", mk_id, ret); + pr_err("Instance %d: no force-halt targets: %d\n", + instance->id, ret); mk_cpu_set_free(targets); - mk_instance_put(instance); return ret; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + pr_info("Force halting multikernel instance %d via NMI\n", + instance->id); ret = mk_arm_force_halt(instance); if (ret) @@ -1906,7 +1929,8 @@ int multikernel_force_halt_by_id(int mk_id) cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); /* * A child instance parks on its own context, so wait for it to @@ -1918,12 +1942,59 @@ int multikernel_force_halt_by_id(int mk_id) * fence, so a baseline cannot claim a machine with a CPU still * running the dead host. */ - if (instance == host_instance) + if (instance == host_instance) { ret = mk_confirm_fenced(instance, targets); - else - mk_instance_settle_halted(instance); + } else { + ret = mk_instance_confirm_parked(instance); + if (ret) + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + else + mk_instance_settle_halted(instance); + } mk_cpu_set_free(targets); + return ret; +} + +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + mk_ipi_endpoint_close(instance); + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; +} + +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by arming the persistent force-halt + * marker and sending NMIs directly to each CPU. The NMI handler checks the + * marker and parks the CPU if it is set. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); mk_instance_put(instance); return ret; } @@ -1962,6 +2033,8 @@ static int __init multikernel_init(void) return ret; } + mk_ipi_handlers_enable(); + pr_info("Multikernel support initialized\n"); return 0; } diff --git a/kernel/multikernel/cpuset.c b/kernel/multikernel/cpuset.c index ad36a4f94fc65a..f3a1a918fe4573 100644 --- a/kernel/multikernel/cpuset.c +++ b/kernel/multikernel/cpuset.c @@ -13,31 +13,22 @@ #include #include -struct mk_cpu_set *mk_cpu_set_alloc(void) -{ - return kzalloc(sizeof(struct mk_cpu_set), GFP_KERNEL); -} - -void mk_cpu_set_free(struct mk_cpu_set *set) +static void mk_cpu_set_lock(const struct mk_cpu_set *set, unsigned long *flags) { - if (!set) - return; - - kfree(set->ids); - kfree(set); + raw_spin_lock_irqsave((raw_spinlock_t *)&set->lock, *flags); } -void mk_cpu_set_clear(struct mk_cpu_set *set) +static void mk_cpu_set_unlock(const struct mk_cpu_set *set, unsigned long flags) { - if (set) - set->nr = 0; + raw_spin_unlock_irqrestore((raw_spinlock_t *)&set->lock, flags); } -static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) +static int mk_cpu_set_index_locked(const struct mk_cpu_set *set, + mk_phys_cpu_t id) { unsigned int i; - for (i = 0; set && i < set->nr; i++) { + for (i = 0; i < set->nr; i++) { if (set->ids[i] == id) return i; } @@ -45,9 +36,35 @@ static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) return -1; } -bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +struct mk_cpu_set *mk_cpu_set_alloc(void) +{ + struct mk_cpu_set *set; + + set = kzalloc_obj(*set, GFP_KERNEL); + if (set) + raw_spin_lock_init(&set->lock); + return set; +} + +void mk_cpu_set_free(struct mk_cpu_set *set) +{ + if (!set) + return; + + kfree(set->ids); + kfree(set); +} + +void mk_cpu_set_clear(struct mk_cpu_set *set) { - return mk_cpu_set_index(set, id) >= 0; + unsigned long flags; + + if (!set) + return; + + mk_cpu_set_lock(set, &flags); + set->nr = 0; + mk_cpu_set_unlock(set, flags); } /** @@ -61,64 +78,209 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) */ int mk_cpu_set_reserve(struct mk_cpu_set *set, unsigned int extra) { - unsigned int cap = set->nr + extra; - mk_phys_cpu_t *ids; + mk_phys_cpu_t *ids = NULL; + mk_phys_cpu_t *old_ids; + unsigned int cap; + unsigned long flags; - if (cap <= set->cap) - return 0; + if (!set) + return -EINVAL; + + for (;;) { + mk_cpu_set_lock(set, &flags); + cap = set->nr + extra; + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + mk_cpu_set_unlock(set, flags); - cap = max_t(unsigned int, cap, 8); - ids = krealloc_array(set->ids, cap, sizeof(*ids), GFP_KERNEL); - if (!ids) - return -ENOMEM; + cap = max_t(unsigned int, cap, 8); + kfree(ids); + ids = kcalloc(cap, sizeof(*ids), GFP_KERNEL); + if (!ids) + return -ENOMEM; - set->ids = ids; - set->cap = cap; - return 0; + mk_cpu_set_lock(set, &flags); + if (set->nr + extra > cap) { + mk_cpu_set_unlock(set, flags); + continue; + } + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + + memcpy(ids, set->ids, set->nr * sizeof(*ids)); + old_ids = set->ids; + set->ids = ids; + set->cap = cap; + mk_cpu_set_unlock(set, flags); + kfree(old_ids); + return 0; + } } /* Idempotent: adding an ID already in the set succeeds without effect */ int mk_cpu_set_add(struct mk_cpu_set *set, mk_phys_cpu_t id) { + unsigned long flags; int ret; - if (mk_cpu_set_contains(set, id)) - return 0; + if (!set) + return -EINVAL; - ret = mk_cpu_set_reserve(set, 1); - if (ret) - return ret; + for (;;) { + mk_cpu_set_lock(set, &flags); + if (mk_cpu_set_index_locked(set, id) >= 0) { + mk_cpu_set_unlock(set, flags); + return 0; + } + if (set->nr < set->cap) { + set->ids[set->nr++] = id; + mk_cpu_set_unlock(set, flags); + return 0; + } + mk_cpu_set_unlock(set, flags); - set->ids[set->nr++] = id; - return 0; + ret = mk_cpu_set_reserve(set, 1); + if (ret) + return ret; + } } bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id) { - int idx = mk_cpu_set_index(set, id); + unsigned long flags; + int idx; + + if (!set) + return false; - if (idx < 0) + mk_cpu_set_lock(set, &flags); + idx = mk_cpu_set_index_locked(set, id); + if (idx < 0) { + mk_cpu_set_unlock(set, flags); return false; + } memmove(&set->ids[idx], &set->ids[idx + 1], (set->nr - idx - 1) * sizeof(set->ids[0])); set->nr--; + mk_cpu_set_unlock(set, flags); return true; } +bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +{ + unsigned long flags; + bool found; + + if (!set) + return false; + + mk_cpu_set_lock(set, &flags); + found = mk_cpu_set_index_locked(set, id) >= 0; + mk_cpu_set_unlock(set, flags); + return found; +} + +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) +{ + unsigned long flags; + unsigned int nr; + + if (!set) + return 0; + + mk_cpu_set_lock(set, &flags); + nr = set->nr; + mk_cpu_set_unlock(set, flags); + return nr; +} + +bool mk_cpu_set_empty(const struct mk_cpu_set *set) +{ + return mk_cpu_set_count(set) == 0; +} + +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) +{ + unsigned long flags; + mk_phys_cpu_t id; + + if (!set) + return MK_PHYS_CPU_INVALID; + + mk_cpu_set_lock(set, &flags); + id = set->nr ? set->ids[0] : MK_PHYS_CPU_INVALID; + mk_cpu_set_unlock(set, flags); + return id; +} + +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id) +{ + unsigned long flags; + bool found = false; + + if (!set || !id) + return false; + + mk_cpu_set_lock(set, &flags); + if (index < set->nr) { + *id = set->ids[index]; + found = true; + } + mk_cpu_set_unlock(set, flags); + return found; +} + int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) { - unsigned int nr = mk_cpu_set_count(src); + unsigned long src_flags; + unsigned long dst_flags; + mk_phys_cpu_t *ids; + unsigned int nr; int ret; - dst->nr = 0; - ret = mk_cpu_set_reserve(dst, nr); - if (ret) - return ret; + if (!dst || !src) + return -EINVAL; + if (dst == src) + return 0; - memcpy(dst->ids, src->ids, nr * sizeof(dst->ids[0])); - dst->nr = nr; - return 0; + for (;;) { + nr = mk_cpu_set_count(src); + ids = nr ? kmalloc_array(nr, sizeof(*ids), GFP_KERNEL) : NULL; + if (nr && !ids) + return -ENOMEM; + + mk_cpu_set_lock(src, &src_flags); + if (src->nr > nr) { + mk_cpu_set_unlock(src, src_flags); + kfree(ids); + continue; + } + nr = src->nr; + if (nr) + memcpy(ids, src->ids, nr * sizeof(*ids)); + mk_cpu_set_unlock(src, src_flags); + + ret = mk_cpu_set_reserve(dst, nr); + if (ret) { + kfree(ids); + return ret; + } + mk_cpu_set_lock(dst, &dst_flags); + if (nr) + memcpy(dst->ids, ids, nr * sizeof(dst->ids[0])); + dst->nr = nr; + mk_cpu_set_unlock(dst, dst_flags); + kfree(ids); + return 0; + } } /** @@ -127,22 +289,31 @@ int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) * @size: Buffer size * @set: Set to format * - * Writes "none" for an empty set, a comma-separated list of physical + * Writes none for an empty set, a comma-separated list of physical * IDs otherwise. Output is truncated to @size. Returns the number of * characters written. */ int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set) { + unsigned long flags; unsigned int i; int len = 0; - if (mk_cpu_set_empty(set)) + if (!buf || !size) + return 0; + if (!set) return scnprintf(buf, size, "none"); - for (i = 0; i < set->nr; i++) { + mk_cpu_set_lock(set, &flags); + if (!set->nr) { + mk_cpu_set_unlock(set, flags); + return scnprintf(buf, size, "none"); + } + + for (i = 0; i < set->nr && len < size; i++) { len += scnprintf(buf + len, size - len, "%s%llu", i ? "," : "", set->ids[i]); } - + mk_cpu_set_unlock(set, flags); return len; } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 91ff72e0a168d6..1e5fdf743fd374 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -633,6 +633,7 @@ static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 pay * PCI Device Hotplug Operations */ +#if IS_ENABLED(CONFIG_PCI) static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { @@ -802,6 +803,19 @@ static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) return 0; } +#else /* CONFIG_PCI */ +static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, + const char *driver_override, u32 flags) +{ + return -ENODEV; +} + +static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) +{ + return -ENODEV; +} +#endif /* CONFIG_PCI */ + struct mk_device_hotplug_work { struct work_struct work; u16 domain; diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index c0f6ca709c0298..cd4d437ed84137 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -42,6 +43,19 @@ EXPORT_SYMBOL_GPL(mk_self); struct mk_instance *host_instance; EXPORT_SYMBOL_GPL(host_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /** * mk_dt_extract_instance_info() - Extract instance ID and name from DTB * @dtb_data: Device tree blob data @@ -211,6 +225,7 @@ static int __init mk_instance_alloc_ipi(struct mk_instance *instance) instance->id); return -ENOMEM; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -297,6 +312,11 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) return 0; } ipi_size = (size_t)ipi_pages << PAGE_SHIFT; + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small: %zu < %zu\n", ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -316,38 +336,67 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) static int __init mk_restore_host_instance(void) { struct mk_instance *hi; - phys_addr_t host_ipi_phys; - u32 host_ipi_pages; - size_t host_ipi_size; + struct mk_shared_data *shared; + mk_phys_cpu_t parent_cpu; + phys_addr_t halt_phys; + size_t halt_size; + int parent_id; + u32 halt_pages; + int ret; - if (!mk_chosen_ring("host-ipi", &host_ipi_phys, &host_ipi_pages)) { - pr_warn("No host IPI buffer in the boot tree (spawn won't be able to send to host)\n"); + if (!mk_self || !mk_self->ipi_data) { + pr_err("No parent/child IPI link in the boot tree\n"); return -ENOENT; } - host_ipi_size = (size_t)host_ipi_pages << PAGE_SHIFT; - - hi = mk_instance_alloc(0, "host"); + shared = mk_self->ipi_data; + parent_id = READ_ONCE(shared->parent_id); + parent_cpu = READ_ONCE(shared->parent_doorbell_cpu); + if (parent_id < 0 || parent_id == mk_self->id || + parent_cpu == MK_PHYS_CPU_INVALID) + return -EPROTO; + + hi = mk_instance_alloc(parent_id, "host"); if (!hi) return -ENOMEM; - - /* - * The host's owned CPU set is unknown here; ring its doorbell on - * physical CPU 0 without pretending we know what it owns. - */ - hi->ipi_target = 0; - - hi->ipi_data = memremap(host_ipi_phys, host_ipi_size, MEMREMAP_WB); - if (!hi->ipi_data) { - pr_err("Failed to map host IPI buffer at 0x%llx\n", - (unsigned long long)host_ipi_phys); + hi->ipi_target = parent_cpu; + ret = mk_cpu_set_add(hi->cpus, parent_cpu); + if (ret) + goto err_free; + if (!mk_chosen_ring("host-ipi", &halt_phys, &halt_pages)) { + pr_err("No host force-halt area in the boot tree\n"); + ret = -EPROTO; + goto err_free; + } + halt_size = (size_t)halt_pages << PAGE_SHIFT; + if (halt_size < sizeof(struct mk_shared_data) || + halt_phys == mk_self->ipi_phys) { + pr_err("Invalid host force-halt area: phys=0x%llx, pages=%u\n", + (unsigned long long)halt_phys, halt_pages); + ret = -EPROTO; goto err_free; } - hi->ipi_phys = host_ipi_phys; - hi->ipi_pages = host_ipi_pages; + hi->halt_data = memremap(halt_phys, halt_size, MEMREMAP_WB); + if (!hi->halt_data) { + pr_err("Failed to map host force-halt area at 0x%llx\n", + (unsigned long long)halt_phys); + ret = -ENOMEM; + goto err_free; + } + hi->ipi_data = mk_self->ipi_data; + hi->ipi_phys = mk_self->ipi_phys; + hi->ipi_pages = mk_self->ipi_pages; - if (mk_instance_publish(hi)) { - memunmap(hi->ipi_data); + ret = mk_instance_publish(hi); + if (ret) goto err_free; + ret = mk_ipi_endpoint_init(hi, false); + if (ret) { + mutex_lock(&mk_instance_mutex); + idr_remove(&mk_instance_idr, hi->id); + list_del(&hi->list); + mutex_unlock(&mk_instance_mutex); + mk_instance_free(hi); + return ret; } /* The host is running, or this kernel would not be */ mk_instance_set_state(hi, MK_STATE_ACTIVE); @@ -363,15 +412,13 @@ static int __init mk_restore_host_instance(void) host_instance = hi; - pr_info("Restored host IPI buffer: phys=0x%llx, pages=%u\n", - (unsigned long long)host_ipi_phys, host_ipi_pages); - pr_info("Registered host instance (ID 0) for spawn→host communication\n"); + pr_info("Registered parent instance %d on duplex IPI link\n", parent_id); return 0; err_free: mk_instance_free(hi); - return -ENOMEM; + return ret; } /** @@ -394,6 +441,9 @@ int __init mk_instance_restore_from_manifest(void) int instance_id; const char *instance_name; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + if (!mk_manifest_phys()) { pr_info("No manifest available for multikernel DTB restoration\n"); @@ -436,7 +486,7 @@ int __init mk_instance_restore_from_manifest(void) dtb_virt = initial_boot_params; if (!dtb_virt || !of_have_populated_dt()) { pr_err("Boot device tree from the manifest was not unflattened\n"); - return -ENOENT; + mk_manifest_reject_and_park(-ENOENT); } dtb_len = fdt_totalsize(dtb_virt); @@ -445,7 +495,7 @@ int __init mk_instance_restore_from_manifest(void) ret = mk_dt_extract_instance_info(dtb_virt, dtb_len, &instance_id, &instance_name); if (ret) { pr_err("Failed to extract instance info from DTB: %d\n", ret); - return ret; + mk_manifest_reject_and_park(ret); } pr_info("DTB contains instance ID %d, name '%s'\n", instance_id, instance_name); @@ -494,26 +544,33 @@ int __init mk_instance_restore_from_manifest(void) } ret = mk_instance_publish(instance); - if (ret) { - if (instance->ipi_data) - memunmap(instance->ipi_data); + if (ret) goto cleanup_instance; - } mk_self = instance; - if (mk_restore_host_instance()) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); + ret = mk_restore_host_instance(); + if (ret) + mk_manifest_reject_and_park(ret); + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); pr_info("Successfully restored multikernel self instance %d ('%s') from the boot tree (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); return 0; cleanup_instance: + if (instance->ipi_data) + memunmap(instance->ipi_data); mk_instance_free(instance); config_free: mk_dt_config_free(&config); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } @@ -533,6 +590,7 @@ early_initcall(mk_instance_restore_from_manifest); * * Returns: true if probing should proceed, false to skip entirely */ +#if IS_ENABLED(CONFIG_PCI) bool mk_pci_should_probe(struct pci_bus *bus, int devfn) { struct device_node *np; @@ -550,6 +608,7 @@ bool mk_pci_should_probe(struct pci_bus *bus, int devfn) return available; } EXPORT_SYMBOL_GPL(mk_pci_should_probe); +#endif /* CONFIG_PCI */ /* * A netdev's alias is its interface name, the one the device had in the diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 563bad63aa0f7f..fa1c015b936e3b 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -16,6 +16,15 @@ struct mk_instance *mk_instance_alloc(int id, const char *name); int mk_instance_publish(struct mk_instance *instance); void mk_instance_free(struct mk_instance *instance); +/* core.c */ +int mk_instance_force_halt(struct mk_instance *instance); + +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance); +void mk_poll_ipi_messages(void); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index b9a2a74bbf1ca4..b05fffe6019738 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -1,356 +1,330 @@ // SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2025 Multikernel Technologies, Inc. All rights reserved - */ -#include -#include #include -#include -#include -#include -#include -#include #include -#include +#include +#include +#include +#include +#include +#include #include "internal.h" -/* Callback management */ static struct mk_ipi_handler *mk_handlers; -static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); +static DEFINE_RAW_SPINLOCK(mk_handlers_lock); +static LIST_HEAD(mk_ipi_endpoints); +static DEFINE_RAW_SPINLOCK(mk_ipi_endpoints_lock); +static bool mk_handlers_ready; +static DEFINE_RATELIMIT_STATE(mk_ipi_full_rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); -static void mk_ipi_drain_ring(void); +static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) +{ + return READ_ONCE(instance->ipi_data); +} -/* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. - */ -static inline unsigned int mk_ring_idx(unsigned int i) +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance) { - return i & (MK_IPI_RING_SIZE - 1); + struct mk_shared_data *shared = READ_ONCE(instance->halt_data); + + return shared ? shared : mk_instance_ipi_area(instance); } -/** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring - * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. - */ -void mk_ipi_ring_drop_pending(void) +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; + + if (!mk_instance_ipi_area(instance)) + return -ENODEV; + if (endpoint->registered) + return 0; + endpoint->tx = parent_side ? &instance->ipi_data->to_child : + &instance->ipi_data->to_parent; + endpoint->rx = parent_side ? &instance->ipi_data->to_parent : + &instance->ipi_data->to_child; + endpoint->tx_head = 0; + endpoint->rx_tail = 0; + endpoint->tx_enabled = true; + endpoint->rx_dispatching = false; + endpoint->parent_side = parent_side; + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + list_add_tail_rcu(&endpoint->rx_node, &mk_ipi_endpoints); + endpoint->registered = true; + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + return 0; +} + +void mk_ipi_endpoint_close(struct mk_instance *instance) { - struct mk_ipi_ring *ring; - unsigned int head, tail; + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; - if (!mk_self || !mk_self->ipi_data) + if (!endpoint->registered) return; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx_enabled = false; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); +} - ring = &mk_self->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); +void mk_ipi_endpoint_unregister(struct mk_instance *instance) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; + if (!endpoint->registered) + return; + mk_ipi_endpoint_close(instance); + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + if (endpoint->registered) { + list_del_rcu(&endpoint->rx_node); + endpoint->registered = false; + } + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + synchronize_rcu(); +} - atomic_set(&ring->tail, head); +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu) +{ + struct mk_shared_data *shared = mk_instance_ipi_area(instance); + + if (!shared) + return; + mk_ipi_endpoint_unregister(instance); + mk_shared_data_reset(shared); + WRITE_ONCE(shared->parent_id, parent_id); + WRITE_ONCE(shared->child_id, child_id); + WRITE_ONCE(shared->parent_doorbell_cpu, parent_cpu); + WRITE_ONCE(shared->child_doorbell_cpu, child_cpu); + mk_ipi_endpoint_init(instance, true); } -/** - * multikernel_register_handler - Register a callback for multikernel IPI - * @callback: Function to call when IPI is received - * @ctx: Context pointer passed to the callback - * @ipi_type: IPI type this handler should process - * - * Returns pointer to handler on success, NULL on failure - */ -struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, unsigned int ipi_type) +struct mk_ipi_handler * +multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, + unsigned int ipi_type) { struct mk_ipi_handler *handler; unsigned long flags; if (!callback) return NULL; - handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler) return NULL; - handler->callback = callback; handler->context = ctx; handler->ipi_type = ipi_type; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); handler->next = mk_handlers; mk_handlers = handler; raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - return handler; } EXPORT_SYMBOL(multikernel_register_handler); -/** - * multikernel_unregister_handler - Unregister a multikernel IPI callback - * @handler: Handler pointer returned from multikernel_register_handler - */ void multikernel_unregister_handler(struct mk_ipi_handler *handler) { - struct mk_ipi_handler **pp, *p; + struct mk_ipi_handler **pp, *p = NULL; unsigned long flags; if (!handler) return; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); - pp = &mk_handlers; - while ((p = *pp) != NULL) { + for (pp = &mk_handlers; (p = *pp); pp = &p->next) { if (p == handler) { *pp = p->next; break; } - pp = &p->next; } raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - kfree(p); } EXPORT_SYMBOL(multikernel_unregister_handler); -/* - * An instance's IPI area is allocated when its image is loaded; the - * instance pointer is filled in lazily on first use. - */ -static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) -{ - struct mk_shared_data *ipi_data; - - if (instance->ipi_data) - return instance->ipi_data; - - if (!instance->kimage || !instance->kimage->mk_ipi) - return NULL; - - ipi_data = phys_to_virt(instance->kimage->mk_ipi); - if (cmpxchg(&instance->ipi_data, NULL, ipi_data) == NULL) - pr_info("Initialized IPI ring buffer for instance %d: phys=0x%llx\n", - instance->id, (unsigned long long)instance->kimage->mk_ipi); - - return instance->ipi_data; -} - -/** - * mk_arm_force_halt - Post the force-halt marker for an instance - * @instance: Instance about to be NMIed - * - * The instance's CPUs test the marker from their NMI handlers, so it - * must be armed before the NMIs are sent. It stays armed until the - * kexec path has confirmed every CPU parked and wipes the shared area - * for the next run, which is what makes the NMI rescue idempotent: a - * repeat force halt still reaches CPUs an earlier one missed. - * - * Returns 0 on success, -ENODEV if the instance has no shared IPI area. - */ int mk_arm_force_halt(struct mk_instance *instance) { - struct mk_shared_data *ipi_data = mk_instance_ipi_area(instance); + struct mk_shared_data *shared = mk_instance_halt_data(instance); - if (!ipi_data) + if (!shared) return -ENODEV; - - WRITE_ONCE(ipi_data->force_halt, 1); - /* The marker must be visible before the NMIs that test it */ + WRITE_ONCE(shared->force_halt, 1); smp_wmb(); return 0; } -/** - * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID - * @data: Pointer to data to send - * @data_size: Size of data - * @type: User-defined type identifier - * - * This function enqueues data into the target instance's IPI ring buffer - * and sends an IPI to notify the target CPU. - * - * Returns 0 on success, negative error code on failure - */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { + struct mk_ipi_endpoint *endpoint; struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; + struct mk_shared_data *shared; mk_phys_cpu_t target; + unsigned long flags; + u32 idx; + int ret = 0; - if (!instance) - return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); + if (!instance || data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; + endpoint = &instance->ipi_endpoint; + if (!READ_ONCE(endpoint->registered)) + return -ESHUTDOWN; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled) { + ret = -ESHUTDOWN; + goto unlock; } - - target = instance->ipi_target; - if (target == MK_PHYS_CPU_INVALID) - target = mk_cpu_set_first(instance->cpus); + shared = READ_ONCE(instance->ipi_data); + if (!shared) { + ret = -ENODEV; + goto unlock; + } + target = endpoint->parent_side ? mk_cpu_set_first(instance->cpus) : + READ_ONCE(shared->parent_doorbell_cpu); if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPU to receive the IPI\n", instance_id); - mk_instance_put(instance); - return -ENODEV; + ret = -ENODEV; + goto unlock; } - - if (!mk_instance_ipi_area(instance)) { - pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); - return -ENODEV; + if (endpoint->parent_side) + WRITE_ONCE(shared->child_doorbell_cpu, target); + idx = endpoint->tx_head & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->tx->entries[idx]; + /* Pair with the receiver's release when it makes the slot reusable. */ + if (smp_load_acquire(&slot->ready)) { + ret = -ENOSPC; + goto unlock; } - - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; - } - - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); - - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; - - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; - - if (data && data_size > 0) + WRITE_ONCE(slot->sender_cpu, arch_cpu_physical_id(smp_processor_id())); + WRITE_ONCE(slot->type, type); + WRITE_ONCE(slot->data_size, data_size); + if (data_size) memcpy(slot->buffer, data, data_size); + /* Publish all message fields before the receiver observes readiness. */ + smp_store_release(&slot->ready, 1); + endpoint->tx_head++; +unlock: + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + if (!ret) + mk_arch_send_ipi(target); + else if (ret == -ENOSPC && __ratelimit(&mk_ipi_full_rs)) + printk_deferred(KERN_WARNING + "multikernel: IPI ring full for instance %d\n", + instance->id); + return ret; +} - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); - - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance = mk_instance_find(instance_id); + int ret; + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } -static void mk_ipi_drain_ring(void) +int multikernel_send_ipi_data_to_host(void *data, size_t data_size, + unsigned long type) { - struct mk_ipi_data *slot; - struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; - size_t data_size; - int messages_processed = 0; - - if (!mk_self || !mk_self->ipi_data) - return; + struct mk_instance *instance = READ_ONCE(host_instance); - while (1) { - tail = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.head)); + if (!instance) + return -ENODEV; + return mk_send_ipi_data(instance, data, data_size, type); +} +EXPORT_SYMBOL(multikernel_send_ipi_data_to_host); - if (tail == head) - break; +static void mk_ipi_dispatch(struct mk_ipi_data *slot) +{ + struct mk_ipi_handler *handler; + mk_ipi_callback_t callback = NULL; + void *context = NULL; + unsigned long flags; - slot = &mk_self->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (READ_ONCE(slot->data_size) > MK_MAX_DATA_SIZE) + return; + raw_spin_lock_irqsave(&mk_handlers_lock, flags); + for (handler = mk_handlers; handler; handler = handler->next) { + if (handler->ipi_type == READ_ONCE(slot->type)) { + callback = handler->callback; + context = handler->context; break; - - if (data_size > MK_MAX_DATA_SIZE) { - pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - continue; } + } + raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); + if (callback) + callback(slot, context); +} - /* Dispatch to registered handler */ - raw_spin_lock(&mk_handlers_lock); - for (handler = mk_handlers; handler; handler = handler->next) { - if (handler->ipi_type == slot->type && handler->callback) { - mk_ipi_callback_t cb = handler->callback; - void *ctx = handler->context; +static void mk_ipi_drain_endpoint(struct mk_ipi_endpoint *endpoint) +{ + struct mk_ipi_data *slot; + unsigned long flags; + u32 idx; - raw_spin_unlock(&mk_handlers_lock); - cb(slot, ctx); - goto advance_tail; + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + if (endpoint->rx_dispatching) { + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return; + } + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + for (;;) { + idx = endpoint->rx_tail & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->rx->entries[idx]; + /* Pair with the producer's release publication of this slot. */ + if (!smp_load_acquire(&slot->ready)) { + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_dispatching = false; + /* Close the empty-ring handoff race with a new publication. */ + if (smp_load_acquire(&slot->ready)) { + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + continue; } + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return; } - raw_spin_unlock(&mk_handlers_lock); - -advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; + mk_ipi_dispatch(slot); + /* The callback must finish reading before the slot is reusable. */ + smp_store_release(&slot->ready, 0); + endpoint->rx_tail++; } } -/** - * multikernel_interrupt_handler - Handle the multikernel IPI - * - * This function is called when a multikernel IPI is received. - * Messages are drained here, in interrupt context. - */ -static void multikernel_interrupt_handler(void) +static void mk_ipi_drain_all(void) { - if (!mk_self || !mk_self->ipi_data) + struct mk_ipi_endpoint *endpoint; + + if (!READ_ONCE(mk_handlers_ready)) return; + rcu_read_lock(); + list_for_each_entry_rcu(endpoint, &mk_ipi_endpoints, rx_node) + mk_ipi_drain_endpoint(endpoint); + rcu_read_unlock(); +} - /* - * Drain here rather than from irq_work. We are already in interrupt - * context and every handler is safe to call from it, and irq_work - * brings a failure mode with it: the work is a single static - * instance, so if it is ever left pending - its self-IPI lost while - * the CPU was bringing its APIC up, say - every later queue attempt - * is a no-op and the ring never drains again. - */ - mk_ipi_drain_ring(); +void mk_ipi_handlers_enable(void) +{ + WRITE_ONCE(mk_handlers_ready, true); + mk_ipi_drain_all(); +} + +void mk_poll_ipi_messages(void) +{ + unsigned long flags; + + local_irq_save(flags); + mk_ipi_drain_all(); + local_irq_restore(flags); } -/** - * Generic multikernel interrupt handler - called by the IPI vector - * - * This is the function that gets called by the IPI vector handler. - */ void generic_multikernel_interrupt(void) { - multikernel_interrupt_handler(); + mk_ipi_drain_all(); } /** diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index e54e9568d0344d..5ee431bf98facb 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -28,12 +28,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -55,6 +61,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -73,14 +80,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /*