diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..6758247c93fd7e 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF_MK XLF_MULTIKERNEL_IPI +#else +# define XLF_MK 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF_MK cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 6f55eab4b35256..953fe71fc044a7 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -127,7 +127,7 @@ struct mk_spawn_context { u32 target_apic_id; /* Target CPU's APIC ID */ u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ - u32 reserved; /* Padding for alignment */ + u32 abi_magic; /* Host/spawn generation marker */ /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ /* Optional boot data belongs after boot_params, in the zeroed tail. */ @@ -198,7 +198,8 @@ int mk_spawn_cpu(struct mk_instance *instance, int cpu, /* The pool park set (park page, slot, page tables) as base,size pairs */ int mk_pool_park_regions(u64 *pairs, int max); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..7099f7cd167dce 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,7 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI 0x0100 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 1002fbcf2f5d0b..1341501952d81b 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -554,6 +554,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; @@ -747,10 +752,29 @@ static void *bzImage64_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { +#ifdef CONFIG_MULTIKERNEL image->arch.mk_boot_params = bootparam_load_addr; + + /* + * setup_boot_parameters() copies the host subarchitecture. A + * spawn kernel must take the multikernel platform path instead. + */ + params->hdr.hardware_subarch = X86_SUBARCH_MULTIKERNEL; + + /* + * The spawn trampoline enters the compressed kernel directly, + * bypassing purgatory. The x86 boot protocol's 64-bit entry is + * 0x200 bytes from the start of the protected-mode payload. + */ + image->arch.mk_kernel_entry = kernel_load_addr + 0x200; + ret = mk_e820_fill(image->mk_instance, params); if (ret) goto out_free_params; +#else + ret = -EOPNOTSUPP; + goto out_free_params; +#endif } /* Allocate loader specific data */ diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index a7d31a722f33b7..ef5fc7ce6941b1 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -62,12 +62,16 @@ struct elf_kernel_info { /* * Find multikernel entry point from PT_NOTE section. - * Looks for note with name "Linux" and type 0x4d4b ('MK'). + * The note type carries the generation; the descriptor remains one u64. */ -static unsigned long find_multikernel_entry_note(const void *buf, size_t len, - const Elf64_Ehdr *ehdr) +#define MK_VMLINUX_LEGACY_NOTE_TYPE 0x4d4b + +static int find_multikernel_entry_note(const void *buf, size_t len, + const Elf64_Ehdr *ehdr, + unsigned long *entry) { const Elf64_Phdr *phdrs = buf + ehdr->e_phoff; + bool legacy = false; int i; for (i = 0; i < ehdr->e_phnum; i++) { @@ -91,25 +95,34 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (ptr + note_size > end) break; - if (nhdr->n_type == 0x4d4b && - nhdr->n_namesz == 6 && + if (nhdr->n_namesz == 6 && nhdr->n_descsz == sizeof(u64) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const u64 *note_entry; + + note_entry = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + if (nhdr->n_type == MK_VMLINUX_NOTE_TYPE) { + *entry = *note_entry; + pr_info("multikernel: entry=0x%llx\n", + *note_entry); + return 0; + } + if (nhdr->n_type == MK_VMLINUX_LEGACY_NOTE_TYPE) + legacy = true; } ptr += note_size; } } - return 0; + return legacy ? -EPROTONOSUPPORT : -ENOENT; } /* * Parse ELF kernel and extract key information */ -static int kexec_parse_elf_kernel(const void *kernel_buf, unsigned long kernel_len, +static int kexec_parse_elf_kernel(const void *kernel_buf, + unsigned long kernel_len, + bool multikernel, struct elf_kernel_info *info) { const Elf64_Ehdr *ehdr; @@ -159,14 +172,25 @@ static int kexec_parse_elf_kernel(const void *kernel_buf, unsigned long kernel_l * PT_NOTE contains physical offset from load base, not virtual address. * This is the canonical way and survives symbol stripping. */ - info->multikernel_entry = find_multikernel_entry_note(kernel_buf, kernel_len, ehdr); - if (!info->multikernel_entry) { - pr_err("multikernel_startup_64 entry offset not found in PT_NOTE\n"); - return -ENOEXEC; + info->multikernel_entry = 0; + if (multikernel) { + int ret; + + ret = find_multikernel_entry_note(kernel_buf, kernel_len, ehdr, + &info->multikernel_entry); + if (ret == -EPROTONOSUPPORT) + pr_err("legacy vmlinux note type 0x%x is incompatible; expected 0x%x\n", + MK_VMLINUX_LEGACY_NOTE_TYPE, MK_VMLINUX_NOTE_TYPE); + else if (ret) + pr_err("multikernel ABI note type 0x%x not found\n", + MK_VMLINUX_NOTE_TYPE); + if (ret) + return ret == -ENOENT ? -ENOEXEC : ret; + + pr_info("Multikernel entry offset: 0x%lx\n", + info->multikernel_entry); } - pr_info("Multikernel entry offset: 0x%lx\n", info->multikernel_entry); - /* Find lowest load address and calculate total memory needed */ phdr = (const Elf64_Phdr *)(kernel_buf + ehdr->e_phoff); for (i = 0; i < ehdr->e_phnum; i++) { @@ -326,12 +350,13 @@ static void *vmlinux_load(struct kimage *image, char *kernel, .top_down = true }; struct kexec_buf pbuf = { .image = image, .buf_min = MIN_PURGATORY_ADDR, .buf_max = ULONG_MAX, .top_down = true }; + bool multikernel = image->type == KEXEC_TYPE_MULTIKERNEL; int ret; pr_info("Loading ELF vmlinux (type=%d)\n", image->type); /* Parse ELF headers */ - ret = kexec_parse_elf_kernel(kernel, kernel_len, &elf_info); + ret = kexec_parse_elf_kernel(kernel, kernel_len, multikernel, &elf_info); if (ret) { pr_err("Failed to parse ELF kernel: %d\n", ret); return ERR_PTR(ret); @@ -531,12 +556,17 @@ static void *vmlinux_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { +#ifdef CONFIG_MULTIKERNEL ret = mk_e820_fill(image->mk_instance, params); if (ret) { kvfree(ldata->kernel_buf); kfree(ldata); goto out_free_params; } +#else + ret = -EOPNOTSUPP; + goto out_free_params; +#endif } ldata->bootparams_buf = params; diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 5d4705d407b4c4..fcd674b933102c 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -47,9 +47,9 @@ static void __init multikernel_setup_calibration(void) { phys_addr_t ctx_phys = orig_boot_params - offsetof(struct mk_spawn_context, bp); - struct mk_spawn_context *ctx = __va(ctx_phys); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); - if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + if (!ctx || !ctx->boot_tsc_khz) return; multikernel_tsc_khz = ctx->boot_tsc_khz; diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..54890d936e615e 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -269,7 +270,7 @@ SYM_CODE_END(multikernel_secondary_startup) .balign 4 .long 2f - 1f .long 4f - 3f - .long 0x4d4b + .long MK_VMLINUX_NOTE_TYPE 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index c8741c95c1529e..78d9122761ff5b 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -72,6 +72,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * This kernel's own context: where its CPUs park and wake. A spawn's @@ -86,6 +87,19 @@ static struct mk_spawn_context *mk_own_context(void) return mk_pool ? mk_pool->arch.slot : NULL; } +static void *spawn_trampoline_va; +static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} + extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; extern char mk_secondary_trampoline[]; @@ -455,6 +469,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, instance->arch.spawn_ctx->boot_tsc_khz = tsc_khz; instance->arch.spawn_ctx->boot_apic_hz = (unsigned long)lapic_timer_period * HZ; + instance->arch.spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->arch.spawn_ctx); } @@ -677,18 +692,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u32 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; + } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -705,14 +740,28 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; /* * A spawn kernel cannot calibrate against legacy timers because they * belong to the host. Reuse the selected physical CPU's delay and local @@ -748,19 +797,27 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. The copy matters: * the page arrives holding the HOST kernel's trampoline code, and when * the two binaries differ its offsets are wrong for this kernel. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_own_context(); unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -768,32 +825,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - memcpy(__va(ctx->trampoline_phys), multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)__va(ctx->trampoline_phys) & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index 4a4d3f68eaf011..535c6c1fe8a3e7 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -337,6 +337,17 @@ static struct tty_driver *mktty_spawn_driver; static struct mktty_spawn_state mktty_spawn; static struct mk_ipi_handler *mktty_spawn_handler; static struct console mktty_spawn_console; + +static int mktty_spawn_parent_id(void) +{ + if (!mk_self) + return -ENODEV; + if (mk_self->id == 0) + return 0; + if (!host_instance) + return -ENODEV; + return READ_ONCE(host_instance->id); +} static bool mktty_console_registered; static int mktty_spawn_activate(struct tty_port *port, struct tty_struct *tty) @@ -376,11 +387,14 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, size_t count) { struct mktty_message *msg; - size_t sent = 0, chunk; - int ret; + size_t sent = 0, chunk, msg_size; + int ret, parent_id; if (tty->index != 0) return -ENODEV; + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + return parent_id; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) @@ -393,10 +407,10 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, msg->len = chunk; msg->reserved = 0; memcpy(msg->data, buf + sent, chunk); + msg_size = sizeof(*msg) - MKTTY_MAX_DATA + chunk; - ret = multikernel_send_ipi_data(0, msg, - sizeof(*msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + ret = multikernel_send_ipi_data(parent_id, msg, msg_size, + MKTTY_IPI_TYPE); if (ret < 0) { kfree(msg); return sent > 0 ? sent : ret; @@ -445,12 +459,16 @@ static struct mktty_message mktty_console_msg; static DEFINE_SPINLOCK(mktty_console_lock); static void mktty_spawn_console_write(struct console *con, const char *s, - unsigned int count) + unsigned int count) { unsigned long flags; - size_t chunk; + size_t chunk, msg_size; + int parent_id; spin_lock_irqsave(&mktty_console_lock, flags); + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + goto out; while (count > 0) { chunk = min_t(size_t, count, MKTTY_MAX_DATA); mktty_console_msg.type = MKTTY_MSG_OUTPUT; @@ -458,13 +476,14 @@ static void mktty_spawn_console_write(struct console *con, const char *s, mktty_console_msg.len = chunk; mktty_console_msg.reserved = 0; memcpy(mktty_console_msg.data, s, chunk); + msg_size = sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk; - multikernel_send_ipi_data(0, &mktty_console_msg, - sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + (void)multikernel_send_ipi_data_to_host(&mktty_console_msg, + msg_size, MKTTY_IPI_TYPE); s += chunk; count -= chunk; } +out: spin_unlock_irqrestore(&mktty_console_lock, flags); } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 1ec6a0d5170005..09460a911bb9cc 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -16,6 +16,11 @@ #include #include #include +#include +#include + +struct pci_bus; +struct mk_instance; #ifdef CONFIG_MULTIKERNEL #include @@ -40,6 +45,7 @@ typedef u64 mk_phys_cpu_t; #define MK_PHYS_CPU_INVALID (~(mk_phys_cpu_t)0) struct mk_cpu_set { + raw_spinlock_t lock; unsigned int nr; /* Entries in use */ unsigned int cap; /* Allocated capacity */ mk_phys_cpu_t *ids; @@ -55,25 +61,15 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id); int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src); int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set); -static inline unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) -{ - return set ? set->nr : 0; -} - -static inline bool mk_cpu_set_empty(const struct mk_cpu_set *set) -{ - return mk_cpu_set_count(set) == 0; -} - -static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) -{ - return mk_cpu_set_empty(set) ? MK_PHYS_CPU_INVALID : set->ids[0]; -} +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set); +bool mk_cpu_set_empty(const struct mk_cpu_set *set); +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set); +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id); #define mk_cpu_set_for_each(i, id, set) \ for ((i) = 0; \ - (set) && (i) < (set)->nr && \ - (((id) = (set)->ids[(i)]), true); \ + mk_cpu_set_get((set), (i), &(id)); \ (i)++) /** @@ -88,16 +84,15 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + u32 ready; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ - unsigned int type; /* User-defined type identifier */ + unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ char buffer[MK_MAX_DATA_SIZE]; /* Actual data buffer */ }; /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ }; @@ -109,9 +104,10 @@ struct mk_ipi_ring { /* Presence table capacity, in CPUs (see mk_cpu_rank) */ #define MK_PARKED_MAX 512 -/* Shared memory structures - per-instance design */ +/* One duplex link per parent/child pair. Each ring has one kernel producer. */ struct mk_shared_data { - struct mk_ipi_ring ring; /* IPI message ring buffer */ + struct mk_ipi_ring to_child; + struct mk_ipi_ring to_parent; /* * Force-halt marker, host-owned. Armed before the host NMIs the * instance's CPUs and cleared with the rest of this struct when @@ -131,6 +127,40 @@ struct mk_shared_data { * the struct on re-exec. */ u8 parked[MK_PARKED_MAX]; + s32 parent_id; + s32 child_id; + u32 reserved; + u64 parent_doorbell_cpu; + u64 child_doorbell_cpu; +}; + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + unsigned int i; + + for (i = 0; i < MK_IPI_RING_SIZE; i++) + WRITE_ONCE(ring->entries[i].ready, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->to_child); + mk_ipi_ring_reset(&shared->to_parent); + WRITE_ONCE(shared->force_halt, 0); +} + +struct mk_ipi_endpoint { + struct mk_ipi_ring *tx; + struct mk_ipi_ring *rx; + raw_spinlock_t tx_lock; + raw_spinlock_t rx_lock; + u32 tx_head; + u32 rx_tail; + bool tx_enabled; + bool rx_dispatching; + bool parent_side; + bool registered; + struct list_head rx_node; }; /* Function pointer type for IPI callbacks */ @@ -172,11 +202,18 @@ void multikernel_unregister_handler(struct mk_ipi_handler *handler); * Returns 0 on success, negative error code on failure */ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type); +int multikernel_send_ipi_data_to_host(void *data, size_t data_size, + unsigned long type); void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side); +void mk_ipi_endpoint_unregister(struct mk_instance *instance); +void mk_ipi_endpoint_close(struct mk_instance *instance); +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu); +void mk_ipi_handlers_enable(void); /* * Multikernel Messaging System @@ -655,6 +692,11 @@ struct mk_instance { struct mk_shared_data *ipi_data; /* IPI shared memory buffer (virtual address) */ phys_addr_t ipi_phys; /* IPI buffer physical address */ u32 ipi_pages; /* IPI buffer size in pages */ + /* + * Separate host self area used when a spawn fences its parent. + * NULL for ordinary child records, whose halt area is @ipi_data. + */ + struct mk_shared_data *halt_data; /* * On a spawn's record of its host: the physical address of the * host's pool wake slot, where the host's CPUs park. A backup @@ -669,6 +711,7 @@ struct mk_instance { * explicitly so routing never masquerades as ownership in @cpus. */ mk_phys_cpu_t ipi_target; + struct mk_ipi_endpoint ipi_endpoint; /* Kexec integration */ struct kimage *kimage; /* Associated kimage object */ @@ -922,6 +965,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); @@ -936,6 +980,7 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); @@ -991,6 +1036,11 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -998,6 +1048,7 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 +/* Bumped whenever the shared-memory layout or message semantics change. */ #define MK_FDT_COMPATIBLE "multikernel-v1" /** @@ -1015,8 +1066,6 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) * the host side, and restore an instance from it on the spawn side. */ - - /** * mk_instance_restore_from_manifest() - Restore this instance from the manifest * @@ -1099,6 +1148,8 @@ void mk_arch_register_cpu(mk_phys_cpu_t phys_id); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86, SDEI or diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..3d9bb5fdeee61e --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* One generation marker shared by the spawn context and image capabilities. */ +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b0002 +#define MK_VMLINUX_NOTE_TYPE 0x4d4b0002 + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index e1d0d55b1a0f46..709838b7dfe1da 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -609,6 +609,12 @@ void kimage_free(struct kimage *image) if (image->type == KEXEC_TYPE_MULTIKERNEL) { unsigned long i; + /* Stop delivery before image-owned shared pages are returned. */ +#ifdef CONFIG_MULTIKERNEL + if (image->mk_instance) + mk_ipi_endpoint_unregister(image->mk_instance); +#endif + for (i = 0; i < image->nr_segments; i++) { void *virt_addr = phys_to_virt(image->segment[i].mem); @@ -1701,6 +1707,12 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1755,10 +1767,11 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1775,32 +1788,36 @@ int multikernel_kexec_by_id(int mk_id) PAGE_ALIGN(sizeof(struct mk_shared_data)) >> PAGE_SHIFT; } - /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. - */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); + /* Reset only this parent/child link, after the old child is parked. */ + if (instance->ipi_data) { + mk_ipi_link_reset(instance, mk_self->id, mk_id, + mk_cpu_set_first(mk_self->cpus), + mk_cpu_set_first(instance->cpus)); + } + rc = mk_arch_spawn_instance(mk_image, instance, cpu); + if (rc) { + mk_ipi_endpoint_close(instance); + goto unlock; + } /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * The instance is running once its CPUs leave the park loop. Publish that + * state before dropping the global kexec lock so another exec cannot race + * this boot while the readiness handshake is pending. */ - mk_ipi_ring_drop_pending(); + rc = mk_instance_set_kexec_active(mk_image->mk_id); + if (rc) { + int abort_ret = mk_instance_abort_spawn(instance); - rc = mk_arch_spawn_instance(mk_image, instance, cpu); - if (rc == 0) { - rc = mk_instance_set_kexec_active(mk_image->mk_id); - if (rc) - pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); + if (abort_ret) + pr_crit("Instance %d activation abort failed: %d\n", + mk_id, abort_ret); + goto unlock; } + kexec_unlock(); + return 0; + unlock: kexec_unlock(); return rc; diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index cec1ea32d83361..e7e242d0feac37 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -17,6 +17,7 @@ config MULTIKERNEL depends on KEXEC_CORE depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE + depends on OF select LIBFDT select OF_DYNAMIC if OF help diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 9de50f787d2737..cecdebdcc36d6c 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -171,6 +171,7 @@ static void mk_instance_release(struct kref *kref) pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); + mk_ipi_endpoint_unregister(instance); mk_instance_return_all_cpus(instance); mk_instance_return_pci_devices(instance); @@ -178,6 +179,8 @@ static void mk_instance_release(struct kref *kref) mk_instance_free_memory(instance); mk_instance_track_dump(instance, instance->state, MK_STATE_READY); + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -231,6 +234,9 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) instance->state = MK_STATE_READY; instance->ipi_target = MK_PHYS_CPU_INVALID; + raw_spin_lock_init(&instance->ipi_endpoint.tx_lock); + raw_spin_lock_init(&instance->ipi_endpoint.rx_lock); + INIT_LIST_HEAD(&instance->ipi_endpoint.rx_node); INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); @@ -297,6 +303,8 @@ void mk_instance_free(struct mk_instance *instance) list_del(&plat_dev->list); kfree(plat_dev); } + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -423,12 +431,25 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; /* Empty until the instance first ran, so nothing of it is executing */ - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + if (!instance->cpus_on_slot) + return 0; + + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } + + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -436,6 +457,7 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -1542,7 +1564,12 @@ static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) */ void __noreturn mk_halt_to_pool(void) { - mk_notify_down_and_park(0, MK_SYS_HALTED); + int parent_id; + + if (!host_instance) + panic("multikernel: spawned kernel has no parent instance"); + parent_id = READ_ONCE(host_instance->id); + mk_notify_down_and_park(parent_id, MK_SYS_HALTED); } static void mk_shutdown_work_fn(struct work_struct *work) @@ -1717,7 +1744,7 @@ int multikernel_halt_by_id(int mk_id) return ret; } -/** +/* * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI * @mk_id: Instance ID to halt * @@ -1788,7 +1815,7 @@ EXPORT_SYMBOL_GPL(mk_cpu_rank); static int mk_fence_missing(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing = 0; @@ -1813,7 +1840,7 @@ static int mk_fence_missing(struct mk_instance *instance, static int mk_confirm_fenced(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing, ret; @@ -1847,22 +1874,20 @@ static int mk_confirm_fenced(struct mk_instance *instance, return ret; } -int multikernel_force_halt_by_id(int mk_id) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { - struct mk_instance *instance; struct mk_cpu_set *targets; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; + return -EINVAL; if (instance == mk_self) { - pr_err("Cannot force halt this kernel (id %d)\n", mk_id); - mk_instance_put(instance); + pr_err("Cannot force halt this kernel (id %d)\n", instance->id); return -EINVAL; } @@ -1873,29 +1898,27 @@ int multikernel_force_halt_by_id(int mk_id) * a rerun the instance is stuck for good. The parent is ACTIVE. */ if (instance->state != MK_STATE_ACTIVE && - instance->state != MK_STATE_LOADED) { + (!allow_loaded || instance->state != MK_STATE_LOADED)) { pr_err("Instance %d not running (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } targets = mk_cpu_set_alloc(); - if (!targets) { - mk_instance_put(instance); + if (!targets) return -ENOMEM; - } ret = mk_force_halt_targets(instance, targets); if (!ret && mk_cpu_set_empty(targets)) ret = -EINVAL; if (ret) { - pr_err("Instance %d: no force-halt targets: %d\n", mk_id, ret); + pr_err("Instance %d: no force-halt targets: %d\n", + instance->id, ret); mk_cpu_set_free(targets); - mk_instance_put(instance); return ret; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + pr_info("Force halting multikernel instance %d via NMI\n", + instance->id); ret = mk_arm_force_halt(instance); if (ret) @@ -1906,7 +1929,8 @@ int multikernel_force_halt_by_id(int mk_id) cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); /* * A child instance parks on its own context, so wait for it to @@ -1918,12 +1942,59 @@ int multikernel_force_halt_by_id(int mk_id) * fence, so a baseline cannot claim a machine with a CPU still * running the dead host. */ - if (instance == host_instance) + if (instance == host_instance) { ret = mk_confirm_fenced(instance, targets); - else - mk_instance_settle_halted(instance); + } else { + ret = mk_instance_confirm_parked(instance); + if (ret) + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + else + mk_instance_settle_halted(instance); + } mk_cpu_set_free(targets); + return ret; +} + +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + mk_ipi_endpoint_close(instance); + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; +} + +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by arming the persistent force-halt + * marker and sending NMIs directly to each CPU. The NMI handler checks the + * marker and parks the CPU if it is set. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); mk_instance_put(instance); return ret; } @@ -1962,6 +2033,8 @@ static int __init multikernel_init(void) return ret; } + mk_ipi_handlers_enable(); + pr_info("Multikernel support initialized\n"); return 0; } diff --git a/kernel/multikernel/cpuset.c b/kernel/multikernel/cpuset.c index ad36a4f94fc65a..f3a1a918fe4573 100644 --- a/kernel/multikernel/cpuset.c +++ b/kernel/multikernel/cpuset.c @@ -13,31 +13,22 @@ #include #include -struct mk_cpu_set *mk_cpu_set_alloc(void) -{ - return kzalloc(sizeof(struct mk_cpu_set), GFP_KERNEL); -} - -void mk_cpu_set_free(struct mk_cpu_set *set) +static void mk_cpu_set_lock(const struct mk_cpu_set *set, unsigned long *flags) { - if (!set) - return; - - kfree(set->ids); - kfree(set); + raw_spin_lock_irqsave((raw_spinlock_t *)&set->lock, *flags); } -void mk_cpu_set_clear(struct mk_cpu_set *set) +static void mk_cpu_set_unlock(const struct mk_cpu_set *set, unsigned long flags) { - if (set) - set->nr = 0; + raw_spin_unlock_irqrestore((raw_spinlock_t *)&set->lock, flags); } -static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) +static int mk_cpu_set_index_locked(const struct mk_cpu_set *set, + mk_phys_cpu_t id) { unsigned int i; - for (i = 0; set && i < set->nr; i++) { + for (i = 0; i < set->nr; i++) { if (set->ids[i] == id) return i; } @@ -45,9 +36,35 @@ static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) return -1; } -bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +struct mk_cpu_set *mk_cpu_set_alloc(void) +{ + struct mk_cpu_set *set; + + set = kzalloc_obj(*set, GFP_KERNEL); + if (set) + raw_spin_lock_init(&set->lock); + return set; +} + +void mk_cpu_set_free(struct mk_cpu_set *set) +{ + if (!set) + return; + + kfree(set->ids); + kfree(set); +} + +void mk_cpu_set_clear(struct mk_cpu_set *set) { - return mk_cpu_set_index(set, id) >= 0; + unsigned long flags; + + if (!set) + return; + + mk_cpu_set_lock(set, &flags); + set->nr = 0; + mk_cpu_set_unlock(set, flags); } /** @@ -61,64 +78,209 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) */ int mk_cpu_set_reserve(struct mk_cpu_set *set, unsigned int extra) { - unsigned int cap = set->nr + extra; - mk_phys_cpu_t *ids; + mk_phys_cpu_t *ids = NULL; + mk_phys_cpu_t *old_ids; + unsigned int cap; + unsigned long flags; - if (cap <= set->cap) - return 0; + if (!set) + return -EINVAL; + + for (;;) { + mk_cpu_set_lock(set, &flags); + cap = set->nr + extra; + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + mk_cpu_set_unlock(set, flags); - cap = max_t(unsigned int, cap, 8); - ids = krealloc_array(set->ids, cap, sizeof(*ids), GFP_KERNEL); - if (!ids) - return -ENOMEM; + cap = max_t(unsigned int, cap, 8); + kfree(ids); + ids = kcalloc(cap, sizeof(*ids), GFP_KERNEL); + if (!ids) + return -ENOMEM; - set->ids = ids; - set->cap = cap; - return 0; + mk_cpu_set_lock(set, &flags); + if (set->nr + extra > cap) { + mk_cpu_set_unlock(set, flags); + continue; + } + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + + memcpy(ids, set->ids, set->nr * sizeof(*ids)); + old_ids = set->ids; + set->ids = ids; + set->cap = cap; + mk_cpu_set_unlock(set, flags); + kfree(old_ids); + return 0; + } } /* Idempotent: adding an ID already in the set succeeds without effect */ int mk_cpu_set_add(struct mk_cpu_set *set, mk_phys_cpu_t id) { + unsigned long flags; int ret; - if (mk_cpu_set_contains(set, id)) - return 0; + if (!set) + return -EINVAL; - ret = mk_cpu_set_reserve(set, 1); - if (ret) - return ret; + for (;;) { + mk_cpu_set_lock(set, &flags); + if (mk_cpu_set_index_locked(set, id) >= 0) { + mk_cpu_set_unlock(set, flags); + return 0; + } + if (set->nr < set->cap) { + set->ids[set->nr++] = id; + mk_cpu_set_unlock(set, flags); + return 0; + } + mk_cpu_set_unlock(set, flags); - set->ids[set->nr++] = id; - return 0; + ret = mk_cpu_set_reserve(set, 1); + if (ret) + return ret; + } } bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id) { - int idx = mk_cpu_set_index(set, id); + unsigned long flags; + int idx; + + if (!set) + return false; - if (idx < 0) + mk_cpu_set_lock(set, &flags); + idx = mk_cpu_set_index_locked(set, id); + if (idx < 0) { + mk_cpu_set_unlock(set, flags); return false; + } memmove(&set->ids[idx], &set->ids[idx + 1], (set->nr - idx - 1) * sizeof(set->ids[0])); set->nr--; + mk_cpu_set_unlock(set, flags); return true; } +bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +{ + unsigned long flags; + bool found; + + if (!set) + return false; + + mk_cpu_set_lock(set, &flags); + found = mk_cpu_set_index_locked(set, id) >= 0; + mk_cpu_set_unlock(set, flags); + return found; +} + +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) +{ + unsigned long flags; + unsigned int nr; + + if (!set) + return 0; + + mk_cpu_set_lock(set, &flags); + nr = set->nr; + mk_cpu_set_unlock(set, flags); + return nr; +} + +bool mk_cpu_set_empty(const struct mk_cpu_set *set) +{ + return mk_cpu_set_count(set) == 0; +} + +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) +{ + unsigned long flags; + mk_phys_cpu_t id; + + if (!set) + return MK_PHYS_CPU_INVALID; + + mk_cpu_set_lock(set, &flags); + id = set->nr ? set->ids[0] : MK_PHYS_CPU_INVALID; + mk_cpu_set_unlock(set, flags); + return id; +} + +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id) +{ + unsigned long flags; + bool found = false; + + if (!set || !id) + return false; + + mk_cpu_set_lock(set, &flags); + if (index < set->nr) { + *id = set->ids[index]; + found = true; + } + mk_cpu_set_unlock(set, flags); + return found; +} + int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) { - unsigned int nr = mk_cpu_set_count(src); + unsigned long src_flags; + unsigned long dst_flags; + mk_phys_cpu_t *ids; + unsigned int nr; int ret; - dst->nr = 0; - ret = mk_cpu_set_reserve(dst, nr); - if (ret) - return ret; + if (!dst || !src) + return -EINVAL; + if (dst == src) + return 0; - memcpy(dst->ids, src->ids, nr * sizeof(dst->ids[0])); - dst->nr = nr; - return 0; + for (;;) { + nr = mk_cpu_set_count(src); + ids = nr ? kmalloc_array(nr, sizeof(*ids), GFP_KERNEL) : NULL; + if (nr && !ids) + return -ENOMEM; + + mk_cpu_set_lock(src, &src_flags); + if (src->nr > nr) { + mk_cpu_set_unlock(src, src_flags); + kfree(ids); + continue; + } + nr = src->nr; + if (nr) + memcpy(ids, src->ids, nr * sizeof(*ids)); + mk_cpu_set_unlock(src, src_flags); + + ret = mk_cpu_set_reserve(dst, nr); + if (ret) { + kfree(ids); + return ret; + } + mk_cpu_set_lock(dst, &dst_flags); + if (nr) + memcpy(dst->ids, ids, nr * sizeof(dst->ids[0])); + dst->nr = nr; + mk_cpu_set_unlock(dst, dst_flags); + kfree(ids); + return 0; + } } /** @@ -127,22 +289,31 @@ int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) * @size: Buffer size * @set: Set to format * - * Writes "none" for an empty set, a comma-separated list of physical + * Writes none for an empty set, a comma-separated list of physical * IDs otherwise. Output is truncated to @size. Returns the number of * characters written. */ int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set) { + unsigned long flags; unsigned int i; int len = 0; - if (mk_cpu_set_empty(set)) + if (!buf || !size) + return 0; + if (!set) return scnprintf(buf, size, "none"); - for (i = 0; i < set->nr; i++) { + mk_cpu_set_lock(set, &flags); + if (!set->nr) { + mk_cpu_set_unlock(set, flags); + return scnprintf(buf, size, "none"); + } + + for (i = 0; i < set->nr && len < size; i++) { len += scnprintf(buf + len, size - len, "%s%llu", i ? "," : "", set->ids[i]); } - + mk_cpu_set_unlock(set, flags); return len; } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 91ff72e0a168d6..1e5fdf743fd374 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -633,6 +633,7 @@ static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 pay * PCI Device Hotplug Operations */ +#if IS_ENABLED(CONFIG_PCI) static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { @@ -802,6 +803,19 @@ static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) return 0; } +#else /* CONFIG_PCI */ +static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, + const char *driver_override, u32 flags) +{ + return -ENODEV; +} + +static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) +{ + return -ENODEV; +} +#endif /* CONFIG_PCI */ + struct mk_device_hotplug_work { struct work_struct work; u16 domain; diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index c0f6ca709c0298..cd4d437ed84137 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -42,6 +43,19 @@ EXPORT_SYMBOL_GPL(mk_self); struct mk_instance *host_instance; EXPORT_SYMBOL_GPL(host_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /** * mk_dt_extract_instance_info() - Extract instance ID and name from DTB * @dtb_data: Device tree blob data @@ -211,6 +225,7 @@ static int __init mk_instance_alloc_ipi(struct mk_instance *instance) instance->id); return -ENOMEM; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -297,6 +312,11 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) return 0; } ipi_size = (size_t)ipi_pages << PAGE_SHIFT; + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small: %zu < %zu\n", ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -316,38 +336,67 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) static int __init mk_restore_host_instance(void) { struct mk_instance *hi; - phys_addr_t host_ipi_phys; - u32 host_ipi_pages; - size_t host_ipi_size; + struct mk_shared_data *shared; + mk_phys_cpu_t parent_cpu; + phys_addr_t halt_phys; + size_t halt_size; + int parent_id; + u32 halt_pages; + int ret; - if (!mk_chosen_ring("host-ipi", &host_ipi_phys, &host_ipi_pages)) { - pr_warn("No host IPI buffer in the boot tree (spawn won't be able to send to host)\n"); + if (!mk_self || !mk_self->ipi_data) { + pr_err("No parent/child IPI link in the boot tree\n"); return -ENOENT; } - host_ipi_size = (size_t)host_ipi_pages << PAGE_SHIFT; - - hi = mk_instance_alloc(0, "host"); + shared = mk_self->ipi_data; + parent_id = READ_ONCE(shared->parent_id); + parent_cpu = READ_ONCE(shared->parent_doorbell_cpu); + if (parent_id < 0 || parent_id == mk_self->id || + parent_cpu == MK_PHYS_CPU_INVALID) + return -EPROTO; + + hi = mk_instance_alloc(parent_id, "host"); if (!hi) return -ENOMEM; - - /* - * The host's owned CPU set is unknown here; ring its doorbell on - * physical CPU 0 without pretending we know what it owns. - */ - hi->ipi_target = 0; - - hi->ipi_data = memremap(host_ipi_phys, host_ipi_size, MEMREMAP_WB); - if (!hi->ipi_data) { - pr_err("Failed to map host IPI buffer at 0x%llx\n", - (unsigned long long)host_ipi_phys); + hi->ipi_target = parent_cpu; + ret = mk_cpu_set_add(hi->cpus, parent_cpu); + if (ret) + goto err_free; + if (!mk_chosen_ring("host-ipi", &halt_phys, &halt_pages)) { + pr_err("No host force-halt area in the boot tree\n"); + ret = -EPROTO; + goto err_free; + } + halt_size = (size_t)halt_pages << PAGE_SHIFT; + if (halt_size < sizeof(struct mk_shared_data) || + halt_phys == mk_self->ipi_phys) { + pr_err("Invalid host force-halt area: phys=0x%llx, pages=%u\n", + (unsigned long long)halt_phys, halt_pages); + ret = -EPROTO; goto err_free; } - hi->ipi_phys = host_ipi_phys; - hi->ipi_pages = host_ipi_pages; + hi->halt_data = memremap(halt_phys, halt_size, MEMREMAP_WB); + if (!hi->halt_data) { + pr_err("Failed to map host force-halt area at 0x%llx\n", + (unsigned long long)halt_phys); + ret = -ENOMEM; + goto err_free; + } + hi->ipi_data = mk_self->ipi_data; + hi->ipi_phys = mk_self->ipi_phys; + hi->ipi_pages = mk_self->ipi_pages; - if (mk_instance_publish(hi)) { - memunmap(hi->ipi_data); + ret = mk_instance_publish(hi); + if (ret) goto err_free; + ret = mk_ipi_endpoint_init(hi, false); + if (ret) { + mutex_lock(&mk_instance_mutex); + idr_remove(&mk_instance_idr, hi->id); + list_del(&hi->list); + mutex_unlock(&mk_instance_mutex); + mk_instance_free(hi); + return ret; } /* The host is running, or this kernel would not be */ mk_instance_set_state(hi, MK_STATE_ACTIVE); @@ -363,15 +412,13 @@ static int __init mk_restore_host_instance(void) host_instance = hi; - pr_info("Restored host IPI buffer: phys=0x%llx, pages=%u\n", - (unsigned long long)host_ipi_phys, host_ipi_pages); - pr_info("Registered host instance (ID 0) for spawn→host communication\n"); + pr_info("Registered parent instance %d on duplex IPI link\n", parent_id); return 0; err_free: mk_instance_free(hi); - return -ENOMEM; + return ret; } /** @@ -394,6 +441,9 @@ int __init mk_instance_restore_from_manifest(void) int instance_id; const char *instance_name; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + if (!mk_manifest_phys()) { pr_info("No manifest available for multikernel DTB restoration\n"); @@ -436,7 +486,7 @@ int __init mk_instance_restore_from_manifest(void) dtb_virt = initial_boot_params; if (!dtb_virt || !of_have_populated_dt()) { pr_err("Boot device tree from the manifest was not unflattened\n"); - return -ENOENT; + mk_manifest_reject_and_park(-ENOENT); } dtb_len = fdt_totalsize(dtb_virt); @@ -445,7 +495,7 @@ int __init mk_instance_restore_from_manifest(void) ret = mk_dt_extract_instance_info(dtb_virt, dtb_len, &instance_id, &instance_name); if (ret) { pr_err("Failed to extract instance info from DTB: %d\n", ret); - return ret; + mk_manifest_reject_and_park(ret); } pr_info("DTB contains instance ID %d, name '%s'\n", instance_id, instance_name); @@ -494,26 +544,33 @@ int __init mk_instance_restore_from_manifest(void) } ret = mk_instance_publish(instance); - if (ret) { - if (instance->ipi_data) - memunmap(instance->ipi_data); + if (ret) goto cleanup_instance; - } mk_self = instance; - if (mk_restore_host_instance()) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); + ret = mk_restore_host_instance(); + if (ret) + mk_manifest_reject_and_park(ret); + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); pr_info("Successfully restored multikernel self instance %d ('%s') from the boot tree (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); return 0; cleanup_instance: + if (instance->ipi_data) + memunmap(instance->ipi_data); mk_instance_free(instance); config_free: mk_dt_config_free(&config); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } @@ -533,6 +590,7 @@ early_initcall(mk_instance_restore_from_manifest); * * Returns: true if probing should proceed, false to skip entirely */ +#if IS_ENABLED(CONFIG_PCI) bool mk_pci_should_probe(struct pci_bus *bus, int devfn) { struct device_node *np; @@ -550,6 +608,7 @@ bool mk_pci_should_probe(struct pci_bus *bus, int devfn) return available; } EXPORT_SYMBOL_GPL(mk_pci_should_probe); +#endif /* CONFIG_PCI */ /* * A netdev's alias is its interface name, the one the device had in the diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 563bad63aa0f7f..fa1c015b936e3b 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -16,6 +16,15 @@ struct mk_instance *mk_instance_alloc(int id, const char *name); int mk_instance_publish(struct mk_instance *instance); void mk_instance_free(struct mk_instance *instance); +/* core.c */ +int mk_instance_force_halt(struct mk_instance *instance); + +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance); +void mk_poll_ipi_messages(void); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index b9a2a74bbf1ca4..b05fffe6019738 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -1,356 +1,330 @@ // SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2025 Multikernel Technologies, Inc. All rights reserved - */ -#include -#include #include -#include -#include -#include -#include -#include #include -#include +#include +#include +#include +#include +#include +#include #include "internal.h" -/* Callback management */ static struct mk_ipi_handler *mk_handlers; -static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); +static DEFINE_RAW_SPINLOCK(mk_handlers_lock); +static LIST_HEAD(mk_ipi_endpoints); +static DEFINE_RAW_SPINLOCK(mk_ipi_endpoints_lock); +static bool mk_handlers_ready; +static DEFINE_RATELIMIT_STATE(mk_ipi_full_rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); -static void mk_ipi_drain_ring(void); +static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) +{ + return READ_ONCE(instance->ipi_data); +} -/* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. - */ -static inline unsigned int mk_ring_idx(unsigned int i) +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance) { - return i & (MK_IPI_RING_SIZE - 1); + struct mk_shared_data *shared = READ_ONCE(instance->halt_data); + + return shared ? shared : mk_instance_ipi_area(instance); } -/** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring - * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. - */ -void mk_ipi_ring_drop_pending(void) +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; + + if (!mk_instance_ipi_area(instance)) + return -ENODEV; + if (endpoint->registered) + return 0; + endpoint->tx = parent_side ? &instance->ipi_data->to_child : + &instance->ipi_data->to_parent; + endpoint->rx = parent_side ? &instance->ipi_data->to_parent : + &instance->ipi_data->to_child; + endpoint->tx_head = 0; + endpoint->rx_tail = 0; + endpoint->tx_enabled = true; + endpoint->rx_dispatching = false; + endpoint->parent_side = parent_side; + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + list_add_tail_rcu(&endpoint->rx_node, &mk_ipi_endpoints); + endpoint->registered = true; + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + return 0; +} + +void mk_ipi_endpoint_close(struct mk_instance *instance) { - struct mk_ipi_ring *ring; - unsigned int head, tail; + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; - if (!mk_self || !mk_self->ipi_data) + if (!endpoint->registered) return; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx_enabled = false; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); +} - ring = &mk_self->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); +void mk_ipi_endpoint_unregister(struct mk_instance *instance) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; + if (!endpoint->registered) + return; + mk_ipi_endpoint_close(instance); + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + if (endpoint->registered) { + list_del_rcu(&endpoint->rx_node); + endpoint->registered = false; + } + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + synchronize_rcu(); +} - atomic_set(&ring->tail, head); +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu) +{ + struct mk_shared_data *shared = mk_instance_ipi_area(instance); + + if (!shared) + return; + mk_ipi_endpoint_unregister(instance); + mk_shared_data_reset(shared); + WRITE_ONCE(shared->parent_id, parent_id); + WRITE_ONCE(shared->child_id, child_id); + WRITE_ONCE(shared->parent_doorbell_cpu, parent_cpu); + WRITE_ONCE(shared->child_doorbell_cpu, child_cpu); + mk_ipi_endpoint_init(instance, true); } -/** - * multikernel_register_handler - Register a callback for multikernel IPI - * @callback: Function to call when IPI is received - * @ctx: Context pointer passed to the callback - * @ipi_type: IPI type this handler should process - * - * Returns pointer to handler on success, NULL on failure - */ -struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, unsigned int ipi_type) +struct mk_ipi_handler * +multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, + unsigned int ipi_type) { struct mk_ipi_handler *handler; unsigned long flags; if (!callback) return NULL; - handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler) return NULL; - handler->callback = callback; handler->context = ctx; handler->ipi_type = ipi_type; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); handler->next = mk_handlers; mk_handlers = handler; raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - return handler; } EXPORT_SYMBOL(multikernel_register_handler); -/** - * multikernel_unregister_handler - Unregister a multikernel IPI callback - * @handler: Handler pointer returned from multikernel_register_handler - */ void multikernel_unregister_handler(struct mk_ipi_handler *handler) { - struct mk_ipi_handler **pp, *p; + struct mk_ipi_handler **pp, *p = NULL; unsigned long flags; if (!handler) return; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); - pp = &mk_handlers; - while ((p = *pp) != NULL) { + for (pp = &mk_handlers; (p = *pp); pp = &p->next) { if (p == handler) { *pp = p->next; break; } - pp = &p->next; } raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - kfree(p); } EXPORT_SYMBOL(multikernel_unregister_handler); -/* - * An instance's IPI area is allocated when its image is loaded; the - * instance pointer is filled in lazily on first use. - */ -static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) -{ - struct mk_shared_data *ipi_data; - - if (instance->ipi_data) - return instance->ipi_data; - - if (!instance->kimage || !instance->kimage->mk_ipi) - return NULL; - - ipi_data = phys_to_virt(instance->kimage->mk_ipi); - if (cmpxchg(&instance->ipi_data, NULL, ipi_data) == NULL) - pr_info("Initialized IPI ring buffer for instance %d: phys=0x%llx\n", - instance->id, (unsigned long long)instance->kimage->mk_ipi); - - return instance->ipi_data; -} - -/** - * mk_arm_force_halt - Post the force-halt marker for an instance - * @instance: Instance about to be NMIed - * - * The instance's CPUs test the marker from their NMI handlers, so it - * must be armed before the NMIs are sent. It stays armed until the - * kexec path has confirmed every CPU parked and wipes the shared area - * for the next run, which is what makes the NMI rescue idempotent: a - * repeat force halt still reaches CPUs an earlier one missed. - * - * Returns 0 on success, -ENODEV if the instance has no shared IPI area. - */ int mk_arm_force_halt(struct mk_instance *instance) { - struct mk_shared_data *ipi_data = mk_instance_ipi_area(instance); + struct mk_shared_data *shared = mk_instance_halt_data(instance); - if (!ipi_data) + if (!shared) return -ENODEV; - - WRITE_ONCE(ipi_data->force_halt, 1); - /* The marker must be visible before the NMIs that test it */ + WRITE_ONCE(shared->force_halt, 1); smp_wmb(); return 0; } -/** - * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID - * @data: Pointer to data to send - * @data_size: Size of data - * @type: User-defined type identifier - * - * This function enqueues data into the target instance's IPI ring buffer - * and sends an IPI to notify the target CPU. - * - * Returns 0 on success, negative error code on failure - */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { + struct mk_ipi_endpoint *endpoint; struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; + struct mk_shared_data *shared; mk_phys_cpu_t target; + unsigned long flags; + u32 idx; + int ret = 0; - if (!instance) - return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); + if (!instance || data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; + endpoint = &instance->ipi_endpoint; + if (!READ_ONCE(endpoint->registered)) + return -ESHUTDOWN; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled) { + ret = -ESHUTDOWN; + goto unlock; } - - target = instance->ipi_target; - if (target == MK_PHYS_CPU_INVALID) - target = mk_cpu_set_first(instance->cpus); + shared = READ_ONCE(instance->ipi_data); + if (!shared) { + ret = -ENODEV; + goto unlock; + } + target = endpoint->parent_side ? mk_cpu_set_first(instance->cpus) : + READ_ONCE(shared->parent_doorbell_cpu); if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPU to receive the IPI\n", instance_id); - mk_instance_put(instance); - return -ENODEV; + ret = -ENODEV; + goto unlock; } - - if (!mk_instance_ipi_area(instance)) { - pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); - return -ENODEV; + if (endpoint->parent_side) + WRITE_ONCE(shared->child_doorbell_cpu, target); + idx = endpoint->tx_head & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->tx->entries[idx]; + /* Pair with the receiver's release when it makes the slot reusable. */ + if (smp_load_acquire(&slot->ready)) { + ret = -ENOSPC; + goto unlock; } - - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; - } - - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); - - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; - - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; - - if (data && data_size > 0) + WRITE_ONCE(slot->sender_cpu, arch_cpu_physical_id(smp_processor_id())); + WRITE_ONCE(slot->type, type); + WRITE_ONCE(slot->data_size, data_size); + if (data_size) memcpy(slot->buffer, data, data_size); + /* Publish all message fields before the receiver observes readiness. */ + smp_store_release(&slot->ready, 1); + endpoint->tx_head++; +unlock: + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + if (!ret) + mk_arch_send_ipi(target); + else if (ret == -ENOSPC && __ratelimit(&mk_ipi_full_rs)) + printk_deferred(KERN_WARNING + "multikernel: IPI ring full for instance %d\n", + instance->id); + return ret; +} - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); - - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance = mk_instance_find(instance_id); + int ret; + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } -static void mk_ipi_drain_ring(void) +int multikernel_send_ipi_data_to_host(void *data, size_t data_size, + unsigned long type) { - struct mk_ipi_data *slot; - struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; - size_t data_size; - int messages_processed = 0; - - if (!mk_self || !mk_self->ipi_data) - return; + struct mk_instance *instance = READ_ONCE(host_instance); - while (1) { - tail = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.head)); + if (!instance) + return -ENODEV; + return mk_send_ipi_data(instance, data, data_size, type); +} +EXPORT_SYMBOL(multikernel_send_ipi_data_to_host); - if (tail == head) - break; +static void mk_ipi_dispatch(struct mk_ipi_data *slot) +{ + struct mk_ipi_handler *handler; + mk_ipi_callback_t callback = NULL; + void *context = NULL; + unsigned long flags; - slot = &mk_self->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (READ_ONCE(slot->data_size) > MK_MAX_DATA_SIZE) + return; + raw_spin_lock_irqsave(&mk_handlers_lock, flags); + for (handler = mk_handlers; handler; handler = handler->next) { + if (handler->ipi_type == READ_ONCE(slot->type)) { + callback = handler->callback; + context = handler->context; break; - - if (data_size > MK_MAX_DATA_SIZE) { - pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - continue; } + } + raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); + if (callback) + callback(slot, context); +} - /* Dispatch to registered handler */ - raw_spin_lock(&mk_handlers_lock); - for (handler = mk_handlers; handler; handler = handler->next) { - if (handler->ipi_type == slot->type && handler->callback) { - mk_ipi_callback_t cb = handler->callback; - void *ctx = handler->context; +static void mk_ipi_drain_endpoint(struct mk_ipi_endpoint *endpoint) +{ + struct mk_ipi_data *slot; + unsigned long flags; + u32 idx; - raw_spin_unlock(&mk_handlers_lock); - cb(slot, ctx); - goto advance_tail; + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + if (endpoint->rx_dispatching) { + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return; + } + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + for (;;) { + idx = endpoint->rx_tail & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->rx->entries[idx]; + /* Pair with the producer's release publication of this slot. */ + if (!smp_load_acquire(&slot->ready)) { + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_dispatching = false; + /* Close the empty-ring handoff race with a new publication. */ + if (smp_load_acquire(&slot->ready)) { + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + continue; } + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return; } - raw_spin_unlock(&mk_handlers_lock); - -advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; + mk_ipi_dispatch(slot); + /* The callback must finish reading before the slot is reusable. */ + smp_store_release(&slot->ready, 0); + endpoint->rx_tail++; } } -/** - * multikernel_interrupt_handler - Handle the multikernel IPI - * - * This function is called when a multikernel IPI is received. - * Messages are drained here, in interrupt context. - */ -static void multikernel_interrupt_handler(void) +static void mk_ipi_drain_all(void) { - if (!mk_self || !mk_self->ipi_data) + struct mk_ipi_endpoint *endpoint; + + if (!READ_ONCE(mk_handlers_ready)) return; + rcu_read_lock(); + list_for_each_entry_rcu(endpoint, &mk_ipi_endpoints, rx_node) + mk_ipi_drain_endpoint(endpoint); + rcu_read_unlock(); +} - /* - * Drain here rather than from irq_work. We are already in interrupt - * context and every handler is safe to call from it, and irq_work - * brings a failure mode with it: the work is a single static - * instance, so if it is ever left pending - its self-IPI lost while - * the CPU was bringing its APIC up, say - every later queue attempt - * is a no-op and the ring never drains again. - */ - mk_ipi_drain_ring(); +void mk_ipi_handlers_enable(void) +{ + WRITE_ONCE(mk_handlers_ready, true); + mk_ipi_drain_all(); +} + +void mk_poll_ipi_messages(void) +{ + unsigned long flags; + + local_irq_save(flags); + mk_ipi_drain_all(); + local_irq_restore(flags); } -/** - * Generic multikernel interrupt handler - called by the IPI vector - * - * This is the function that gets called by the IPI vector handler. - */ void generic_multikernel_interrupt(void) { - multikernel_interrupt_handler(); + mk_ipi_drain_all(); } /** diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index e54e9568d0344d..5ee431bf98facb 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -28,12 +28,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -55,6 +61,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -73,14 +80,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /*