[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH 01/12 gnumach] i386: Fix lapic and ioapic for smp
|
From: |
Damien Zammit |
|
Subject: |
[PATCH 01/12 gnumach] i386: Fix lapic and ioapic for smp |
|
Date: |
Tue, 31 Jan 2023 09:35:21 +0000 |
Also-by: Almudena Garcia <liberamenso10000@gmail.com>
---
i386/i386/apic.c | 87 ++++++++++++++++++++++++++++++---
i386/i386/apic.h | 114 ++++++++++++++++++++++++++++++++++++++++---
i386/i386/smp.c | 89 ++++++++++++++++++++++++++++++++-
i386/i386/smp.h | 7 +++
i386/i386at/ioapic.c | 97 ++++++++++--------------------------
5 files changed, 307 insertions(+), 87 deletions(-)
diff --git a/i386/i386/apic.c b/i386/i386/apic.c
index d30084e2..e53d4749 100644
--- a/i386/i386/apic.c
+++ b/i386/i386/apic.c
@@ -19,6 +19,8 @@
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111, USA. */
#include <i386/apic.h>
+#include <i386/cpu.h>
+#include <i386at/idt.h>
#include <string.h>
#include <vm/vm_kern.h>
#include <kern/printf.h>
@@ -112,7 +114,7 @@ acpi_get_irq_override(uint8_t pin)
* apic_get_cpu_apic_id: returns the apic_id of a cpu.
* Receives as input the kernel ID of a CPU.
*/
-uint16_t
+int
apic_get_cpu_apic_id(int kernel_id)
{
if (kernel_id >= NCPUS)
@@ -121,6 +123,24 @@ apic_get_cpu_apic_id(int kernel_id)
return apic_data.cpu_lapic_list[kernel_id];
}
+
+/*
+ * apic_get_cpu_kernel_id: returns the kernel_id of a cpu.
+ * Receives as input the APIC ID of a CPU.
+ */
+int
+apic_get_cpu_kernel_id(uint16_t apic_id)
+{
+ int i;
+
+ for (i = 0; i < apic_data.ncpus; i++) {
+ if (apic_data.cpu_lapic_list[i] == apic_id)
+ return i;
+ }
+
+ return -1;
+}
+
/* apic_get_lapic: returns a reference to the common memory address for Local
APIC. */
volatile ApicLocalUnit*
apic_get_lapic(void)
@@ -158,17 +178,13 @@ apic_get_num_ioapics(void)
/*
* apic_get_current_cpu: returns the apic_id of current cpu.
*/
-uint16_t
+int
apic_get_current_cpu(void)
{
- uint16_t apic_id;
-
if(lapic == NULL)
- apic_id = 0;
- else
- apic_id = lapic->apic_id.r;
+ return -1;
- return apic_id;
+ return (lapic->apic_id.r >> 24) & 0xff;
}
@@ -235,6 +251,61 @@ void apic_print_info(void)
}
}
+void apic_send_ipi(unsigned dest_shorthand, unsigned deliv_mode, unsigned
dest_mode, unsigned level, unsigned trig_mode, unsigned vector, unsigned
dest_id)
+{
+ IcrLReg icrl_values;
+ IcrHReg icrh_values;
+
+ icrl_values.destination_shorthand = dest_shorthand;
+ icrl_values.delivery_mode = deliv_mode;
+ icrl_values.destination_mode = dest_mode;
+ icrl_values.level = level;
+ icrl_values.trigger_mode = trig_mode;
+ icrl_values.vector = vector;
+ icrh_values.destination_field = dest_id;
+
+ lapic->icr_high = icrh_values;
+ lapic->icr_low = icrl_values;
+}
+
+void
+lapic_enable(void)
+{
+ unsigned long flags;
+ int apic_id;
+ volatile uint32_t dummy;
+
+ cpu_intr_save(&flags);
+
+ apic_id = apic_get_current_cpu();
+
+ dummy = lapic->dest_format.r;
+ lapic->dest_format.r = 0xffffffff; /* flat model */
+ dummy = lapic->logical_dest.r;
+ lapic->logical_dest.r = lapic->apic_id.r; /* target self */
+ dummy = lapic->lvt_lint0.r;
+ lapic->lvt_lint0.r = dummy | LAPIC_DISABLE;
+ dummy = lapic->lvt_lint1.r;
+ lapic->lvt_lint1.r = dummy | LAPIC_DISABLE;
+ dummy = lapic->lvt_performance_monitor.r;
+ lapic->lvt_performance_monitor.r = dummy | LAPIC_DISABLE;
+ if (apic_id != 0)
+ {
+ dummy = lapic->lvt_timer.r;
+ lapic->lvt_timer.r = dummy | LAPIC_DISABLE;
+ }
+ dummy = lapic->task_pri.r;
+ lapic->task_pri.r = 0;
+
+ /* Enable LAPIC to send or recieve IPI/SIPIs */
+ dummy = lapic->spurious_vector.r;
+ lapic->spurious_vector.r = dummy | LAPIC_ENABLE;
+
+ lapic->error_status.r = 0;
+
+ cpu_intr_restore(flags);
+}
+
void
lapic_eoi(void)
{
diff --git a/i386/i386/apic.h b/i386/i386/apic.h
index 0bb1bd73..ac083d26 100644
--- a/i386/i386/apic.h
+++ b/i386/i386/apic.h
@@ -61,10 +61,99 @@ union ioapic_route_entry_union {
struct ioapic_route_entry both;
};
+
+/* Grateful to trasterlabs for this snippet */
+
+typedef union u_icr_low
+{
+ uint32_t value[4];
+ struct
+ {
+ uint32_t r; // FEE0 0300H - 4 bytes
+ unsigned :32; // FEE0 0304H
+ unsigned :32; // FEE0 0308H
+ unsigned :32; // FEE0 030CH
+ };
+ struct
+ {
+ unsigned vector: 8; /* Vector of interrupt. Lowest 8 bits of routine
address */
+ unsigned delivery_mode : 3;
+ unsigned destination_mode: 1;
+ unsigned delivery_status: 1;
+ unsigned :1;
+ unsigned level: 1;
+ unsigned trigger_mode: 1;
+ unsigned :2;
+ unsigned destination_shorthand: 2;
+ unsigned :12;
+ };
+} IcrLReg;
+
+typedef union u_icr_high
+{
+ uint32_t value[4];
+ struct
+ {
+ uint32_t r; // FEE0 0310H - 4 bytes
+ unsigned :32; // FEE0 0314H
+ unsigned :32; // FEE0 0318H
+ unsigned :32; // FEE0 031CH
+ };
+ struct
+ {
+ unsigned :24; // FEE0 0310H - 4 bytes
+ unsigned destination_field :8; /* APIC ID (in physical mode) or MDA
(in logical) of destination processor */
+ };
+} IcrHReg;
+
+
+typedef enum e_icr_dest_shorthand
+{
+ NO_SHORTHAND = 0,
+ SELF = 1,
+ ALL_INCLUDING_SELF = 2,
+ ALL_EXCLUDING_SELF = 3
+} icr_dest_shorthand;
+
+typedef enum e_icr_deliv_mode
+{
+ FIXED = 0,
+ LOWEST_PRIORITY = 1,
+ SMI = 2,
+ NMI = 4,
+ INIT = 5,
+ STARTUP = 6,
+} icr_deliv_mode;
+
+typedef enum e_icr_dest_mode
+{
+ PHYSICAL = 0,
+ LOGICAL = 1
+} icr_dest_mode;
+
+typedef enum e_icr_deliv_status
+{
+ IDLE = 0,
+ SEND_PENDING = 1
+} icr_deliv_status;
+
+typedef enum e_icr_level
+{
+ DE_ASSERT = 0,
+ ASSERT = 1
+} icr_level;
+
+typedef enum e_irc_trigger_mode
+{
+ EDGE = 0,
+ LEVEL = 1
+} irc_trigger_mode;
+
+
typedef struct ApicLocalUnit {
ApicReg reserved0; /* 0x000 */
ApicReg reserved1; /* 0x010 */
- ApicReg apic_id; /* 0x020 */
+ ApicReg apic_id; /* 0x020. Hardware ID of current
processor */
ApicReg version; /* 0x030 */
ApicReg reserved4; /* 0x040 */
ApicReg reserved5; /* 0x050 */
@@ -84,8 +173,8 @@ typedef struct ApicLocalUnit {
ApicReg error_status; /* 0x280 */
ApicReg reserved28[6]; /* 0x290 */
ApicReg lvt_cmci; /* 0x2f0 */
- ApicReg icr_low; /* 0x300 */
- ApicReg icr_high; /* 0x310 */
+ IcrLReg icr_low; /* 0x300. Store the information to
send an IPI (Inter-processor Interrupt) */
+ IcrHReg icr_high; /* 0x310. Store the IPI destination
*/
ApicReg lvt_timer; /* 0x320 */
ApicReg lvt_thermal; /* 0x330 */
ApicReg lvt_performance_monitor; /* 0x340 */
@@ -138,24 +227,27 @@ void apic_add_cpu(uint16_t apic_id);
void apic_lapic_init(ApicLocalUnit* lapic_ptr);
void apic_add_ioapic(struct IoApicData);
void apic_add_irq_override(struct IrqOverrideData irq_over);
+void apic_send_ipi(unsigned dest_shorthand, unsigned deliv_mode, unsigned
dest_mode, unsigned level, unsigned trig_mode, unsigned vector, unsigned
dest_id);
IrqOverrideData *acpi_get_irq_override(uint8_t gsi);
-uint16_t apic_get_cpu_apic_id(int kernel_id);
+int apic_get_cpu_apic_id(int kernel_id);
+int apic_get_cpu_kernel_id(uint16_t apic_id);
volatile ApicLocalUnit* apic_get_lapic(void);
struct IoApicData *apic_get_ioapic(int kernel_id);
uint8_t apic_get_numcpus(void);
uint8_t apic_get_num_ioapics(void);
-uint16_t apic_get_current_cpu(void);
+int apic_get_current_cpu(void);
void apic_print_info(void);
int apic_refit_cpulist(void);
void picdisable(void);
void lapic_eoi(void);
void ioapic_irq_eoi(int pin);
+void lapic_enable(void);
void lapic_enable_timer(void);
void ioapic_mask_irqs(void);
void ioapic_toggle(int pin, int mask);
void ioapic_configure(void);
-extern int timer_pin;
+extern int duplicate_pin;
extern void intnull(int unit);
extern volatile ApicLocalUnit* lapic;
@@ -172,9 +264,13 @@ extern volatile ApicLocalUnit* lapic;
# define IMCR_USE_PIC 0
# define IMCR_USE_APIC 1
+#define LAPIC_LOW_PRIO 0x100
+#define LAPIC_NMI 0x400
+#define LAPIC_EXTINT 0x700
+#define LAPIC_LEVEL_TRIGGERED 0x8000
+
#define LAPIC_ENABLE 0x100
#define LAPIC_FOCUS 0x200
-#define LAPIC_NMI 0x400
#define LAPIC_ENABLE_DIRECTED_EOI 0x1000
#define LAPIC_DISABLE 0x10000
#define LAPIC_TIMER_PERIODIC 0x20000
@@ -198,6 +294,10 @@ extern volatile ApicLocalUnit* lapic;
#define IOAPIC_MASK_ENABLED 0
#define IOAPIC_MASK_DISABLED 1
+#define APIC_MSR 0x1b
+#define APIC_MSR_BSP 0x100 /* Processor is a BSP */
+#define APIC_MSR_ENABLE 0x800
+
/* Set or clear a bit in a 255-bit APIC mask register.
These registers are spread through eight 32-bit registers. */
#define APIC_SET_MASK_BIT(reg, bit) \
diff --git a/i386/i386/smp.c b/i386/i386/smp.c
index d7523a73..c351efaa 100644
--- a/i386/i386/smp.c
+++ b/i386/i386/smp.c
@@ -18,11 +18,18 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111, USA. */
-#include <i386/i386/apic.h>
-#include <i386/i386/smp.h>
+#include <i386/apic.h>
+#include <i386/smp.h>
+#include <i386/cpu.h>
+#include <i386/pit.h>
+#include <i386at/idt.h>
+#include <i386at/acpi_parse_apic.h>
+#include <kern/printf.h>
+#include <mach/machine.h>
#include <kern/smp.h>
+#define pause_memory asm volatile ("pause" : : : "memory")
/*
* smp_data_init: initialize smp_data structure
@@ -33,6 +40,84 @@ static void smp_data_init(void)
{
uint8_t numcpus = apic_get_numcpus();
smp_set_numcpus(numcpus);
+
+ for(int i = 0; i < numcpus; i++){
+ machine_slot[i].is_cpu = TRUE;
+ }
+
+}
+
+void smp_pmap_update(unsigned apic_id)
+{
+ unsigned long flags;
+
+ cpu_intr_save(&flags);
+
+ printf("Sending IPI(%u) to call TLB shootdown...", apic_id);
+ apic_send_ipi(NO_SHORTHAND, FIXED, PHYSICAL, ASSERT, EDGE,
CALL_SINGLE_FUNCTION_BASE, apic_id);
+
+ do {
+ pause_memory;
+ } while(lapic->icr_low.delivery_status == SEND_PENDING);
+
+ printf("done\n");
+
+ cpu_intr_restore(flags);
+}
+
+/* See Intel IA32/64 Software Developer's Manual 3A Section 8.4.4.1 */
+void smp_startup_cpu(unsigned apic_id, unsigned vector)
+{
+ /* Clear APIC errors */
+ lapic->error_status.r = 0;
+
+ printf("Sending IPIs to APIC ID %u...", apic_id);
+
+ /* Assert INIT IPI */
+ apic_send_ipi(NO_SHORTHAND, INIT, PHYSICAL, ASSERT, LEVEL, 0, apic_id);
+
+ /* Wait for delivery */
+ do {
+ pause_memory;
+ } while(lapic->icr_low.delivery_status == SEND_PENDING);
+
+ /* Deassert INIT IPI */
+ apic_send_ipi(NO_SHORTHAND, INIT, PHYSICAL, DE_ASSERT, LEVEL, 0, apic_id);
+
+ /* Wait for delivery */
+ do {
+ pause_memory;
+ } while(lapic->icr_low.delivery_status == SEND_PENDING);
+
+ /* Wait 10 msec */
+ pit_mdelay(10);
+
+ /* Clear APIC errors */
+ lapic->error_status.r = 0;
+
+ /* First StartUp IPI */
+ apic_send_ipi(NO_SHORTHAND, STARTUP, PHYSICAL, ASSERT, LEVEL, vector >>
12, apic_id);
+
+ /* Wait 200 usec */
+ pit_udelay(200);
+
+ /* Wait for delivery */
+ do {
+ pause_memory;
+ } while(lapic->icr_low.delivery_status == SEND_PENDING);
+
+ /* Second StartUp IPI */
+ apic_send_ipi(NO_SHORTHAND, STARTUP, PHYSICAL, ASSERT, LEVEL, vector >>
12, apic_id);
+
+ /* Wait 200 usec */
+ pit_udelay(200);
+
+ /* Wait for delivery */
+ do {
+ pause_memory;
+ } while(lapic->icr_low.delivery_status == SEND_PENDING);
+
+ printf("done\n");
}
/*
diff --git a/i386/i386/smp.h b/i386/i386/smp.h
index b36ead08..79337022 100644
--- a/i386/i386/smp.h
+++ b/i386/i386/smp.h
@@ -18,4 +18,11 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111, USA. */
+#ifndef _SMP_H_
+#define _SMP_H_
+
int smp_init(void);
+void smp_pmap_update(unsigned apic_id);
+void smp_startup_cpu(unsigned apic_id, unsigned vector);
+
+#endif
diff --git a/i386/i386at/ioapic.c b/i386/i386at/ioapic.c
index c5eb3536..d4269ef0 100644
--- a/i386/i386at/ioapic.c
+++ b/i386/i386at/ioapic.c
@@ -32,8 +32,7 @@
#include <kern/printf.h>
static int has_irq_specific_eoi = 1; /* FIXME: Assume all machines have this */
-static int timer_gsi;
-int timer_pin;
+int duplicate_pin;
uint32_t lapic_timer_val = 0;
uint32_t calibrated_ticks = 0;
@@ -78,6 +77,7 @@ void
picdisable(void)
{
asm("cli");
+ curr_ipl = SPLHI;
/*
** Disable PIC
@@ -147,40 +147,13 @@ ioapic_toggle_entry(int apic, int pin, int mask)
ioapic_write(apic, APIC_IO_REDIR_LOW(pin), entry.lo);
}
-static void
-cpu_rdmsr(uint32_t msr, uint32_t *lo, uint32_t *hi)
-{
- __asm__ __volatile__("rdmsr" : "=a"(*lo), "=d"(*hi) : "c"(msr));
-}
-
-static void
-cpu_wrmsr(uint32_t msr, uint32_t lo, uint32_t hi)
-{
- __asm__ __volatile__("wrmsr" : : "a"(lo), "d"(hi), "c"(msr));
-}
-
-static void
-global_enable_apic(void)
-{
- uint32_t lo = 0;
- uint32_t hi = 0;
- uint32_t msr = 0x1b;
-
- cpu_rdmsr(msr, &lo, &hi);
-
- if (!(lo & (1 << 11))) {
- lo |= (1 << 11);
- cpu_wrmsr(msr, lo, hi);
- }
-}
-
static uint32_t
pit_measure_apic_hz(void)
{
uint32_t start = 0xffffffff;
- /* Prepare accurate delay for 1/100 seconds */
- pit_prepare_sleep(100);
+ /* Prepare accurate delay for 1/hz seconds */
+ pit_prepare_sleep(hz);
/* Set APIC timer */
lapic->init_count.r = start;
@@ -189,7 +162,7 @@ pit_measure_apic_hz(void)
pit_sleep();
/* Stop APIC timer */
- lapic->lvt_timer.r = LAPIC_DISABLE;
+ lapic->lvt_timer.r |= LAPIC_DISABLE;
return start - lapic->cur_count.r;
}
@@ -203,26 +176,18 @@ void lapic_update_timer(void)
void
lapic_enable_timer(void)
{
- spl_t s;
-
- s = sploff();
- asm("cli");
-
/* Set up counter */
lapic->init_count.r = calibrated_ticks;
lapic->divider_config.r = LAPIC_TIMER_DIVIDE_16;
/* Set the timer to interrupt periodically on remapped timer GSI */
- lapic->lvt_timer.r = (IOAPIC_INT_BASE + timer_gsi) | LAPIC_TIMER_PERIODIC;
+ lapic->lvt_timer.r = IOAPIC_INT_BASE | LAPIC_TIMER_PERIODIC;
/* Some buggy hardware requires this set again */
lapic->divider_config.r = LAPIC_TIMER_DIVIDE_16;
- /* Unmask the remapped timer pin and pin 0 always */
- ioapic_toggle(0, IOAPIC_MASK_ENABLED);
- ioapic_toggle(timer_pin, IOAPIC_MASK_ENABLED);
-
- splon(s);
+ /* Enable interrupts for the first time on BSP */
+ asm("sti");
printf("LAPIC timer configured\n");
}
@@ -239,6 +204,9 @@ ioapic_irq_eoi(int pin)
int apic = 0;
union ioapic_route_entry_union oldentry, entry;
+ if (pin == 0)
+ goto skip_specific_eoi;
+
if (!has_irq_specific_eoi) {
/* Workaround for old IOAPICs with no specific EOI */
@@ -258,6 +226,7 @@ ioapic_irq_eoi(int pin)
ioapic->eoi.r = entry.both.vector;
}
+skip_specific_eoi:
lapic_eoi ();
}
@@ -303,12 +272,12 @@ ioapic_configure(void)
/* Assume first IO APIC maps to GSI base 0 */
int gsi, apic = 0, bsp = 0, pin;
IrqOverrideData *irq_over;
+ int timer_gsi;
/* Disable IOAPIC interrupts and set spurious interrupt */
lapic->spurious_vector.r = IOAPIC_SPURIOUS_BASE;
union ioapic_route_entry_union entry = {{0, 0}};
- union ioapic_route_entry_union timer_entry = {{0, 0}};
entry.both.delvmode = IOAPIC_FIXED;
entry.both.destmode = IOAPIC_PHYSICAL;
@@ -332,16 +301,17 @@ ioapic_configure(void)
if (pin == 0) {
/* Save timer info */
timer_gsi = gsi;
- timer_entry = entry;
} else {
- /* Get the actual timer pin by assuming that the pin
- * with duplicated gsi from pin 0 maps to the timer pin */
+ /* Disable duplicated timer gsi */
if (gsi == timer_gsi) {
- timer_pin = pin;
- /* Remap pin 0 interrupt vector to GSI base
+ duplicate_pin = pin;
+ /* Remap this interrupt pin to GSI base
* so we don't duplicate vectors */
- timer_entry.both.vector = IOAPIC_INT_BASE;
- ioapic_write_entry(apic, 0, timer_entry.both);
+ entry.both.vector = IOAPIC_INT_BASE;
+ ioapic_write_entry(apic, duplicate_pin, entry.both);
+ /* Mask the ioapic pin with deduplicated vector as
+ * we will never use it, since timer is on another gsi */
+ mask_irq(duplicate_pin);
}
}
}
@@ -361,16 +331,7 @@ ioapic_configure(void)
}
/* Start the IO APIC receiving interrupts */
- lapic->apic_id.r = apic_get_cpu_apic_id(bsp);
- lapic->dest_format.r = 0xffffffff; /* flat model */
- lapic->logical_dest.r = 0x01000000; /* target bsp */
- lapic->lvt_timer.r = LAPIC_DISABLE;
- lapic->lvt_performance_monitor.r = LAPIC_NMI;
- lapic->lvt_lint0.r = LAPIC_DISABLE;
- lapic->lvt_lint1.r = LAPIC_DISABLE;
- lapic->task_pri.r = 0;
-
- global_enable_apic();
+ lapic_enable();
/* Enable IOAPIC processor focus */
lapic->spurious_vector.r |= LAPIC_FOCUS;
@@ -381,23 +342,19 @@ ioapic_configure(void)
lapic->spurious_vector.r |= LAPIC_ENABLE_DIRECTED_EOI;
}
- /* Enable IOAPIC interrupts */
- lapic->spurious_vector.r |= LAPIC_ENABLE;
-
/* Set one-shot timer */
lapic->divider_config.r = LAPIC_TIMER_DIVIDE_16;
- lapic->lvt_timer.r = IOAPIC_INT_BASE + timer_gsi;
+ lapic->lvt_timer.r = IOAPIC_INT_BASE;
- /* Measure number of APIC timer ticks in 10ms */
- calibrated_ticks = pit_measure_apic_hz();
+ /* Measure number of APIC timer ticks in 1/hz seconds
+ * but calibrate the timer to expire at rate of hz */
+ calibrated_ticks = pit_measure_apic_hz() * hz;
/* Set up counter later */
lapic->lvt_timer.r = LAPIC_DISABLE;
- /* Install clock interrupt handler on both remapped timer pin and pin 0
- * since nobody knows how all x86 timers are wired up */
+ /* Install clock interrupt handler on pin 0 */
ivect[0] = (interrupt_handler_fn)hardclock;
- ivect[timer_pin] = (interrupt_handler_fn)hardclock;
printf("IOAPIC 0 configured\n");
}
--
2.34.1