When allocating page(s), update all the page directories of all the a…

…ctive processors for kernel space. Syncrhonize between smp initialization code and page allocation code so that page directory entry is not missed. It will help in future to go a CPU online and syncrhronize page directory correctly.

apic.c

@@ -92,7 +92,7 @@ void init_lapic() {
  read_msr(0x1b, &eax, &edx);
  lapic_base_register = eax & 0xfffff000;
 
- build_pagetable((pgd_t*)&_kernel_pg_dir, pgtable, lapic_base_register, lapic_base_register, PAGE_SIZE, PGD_PRESENT | PGD_WRITE, PTE_PRESENT | PTE_WRITE);
+ build_pagetable(0, pgtable, lapic_base_register, lapic_base_register, PAGE_SIZE, PGD_PRESENT | PGD_WRITE, PTE_PRESENT | PTE_WRITE);
 
  printf(KERNEL_INFO, "Local APIC address: %p\n", eax);
 

boot32.S

@@ -32,7 +32,7 @@ _start:
  movl $_kernel_pg_table_0-KERNEL_VIRT_ADDR, %edx
  leal 0x1(%edx), %eax
  movl %eax, _kernel_pg_dir-KERNEL_VIRT_ADDR
- movl %eax, _kernel_pg_dir-KERNEL_VIRT_ADDR+KERNEL_PGDIR_ENTRY
+ movl %eax, _kernel_pg_dir-KERNEL_VIRT_ADDR+KERNEL_PGDIR_OFFSET
 
  // populating page table entries that maps 4MB spaces starting from memory address 0
  // both page directory and page table entry should be located in first 4 GB.

krnlconst.hdr

@@ -11,7 +11,7 @@ E820_MAP_COUNT (BASE_WRITE_ADDR+20)
 E820_MAP_ADDRESS (BASE_WRITE_ADDR+24)
 PROTECTED_MODE_STACK 0x1000
 KERNEL_VIRT_ADDR 0x80000000
-KERNEL_PGDIR_ENTRY (KERNEL_VIRT_ADDR>>20)
+KERNEL_PGDIR_OFFSET (KERNEL_VIRT_ADDR>>20)
 KERNEL_STACK_SIZE 0x2000
 PAGE_SHIFT 12
 PAGE_SIZE (1<<PAGE_SHIFT)

lock.c

@@ -6,6 +6,7 @@
 /* Author: Shoily O Rahman <[email protected]> */
 /* */
 /* Date: Oct 16, 2020 */
+/* Lock order - spinlock_smp->lock_pgd */
 /* */
 /*****************************************************************************/
 
@@ -30,3 +31,4 @@ void spinlock_unlock(spinlock *lock) {
  :
  : "cc", "memory" );
 }
+

lock.h

@@ -12,11 +12,14 @@
 #ifndef _LOCK_H
 #define _LOCK_H
 
+#include "util.h"
+#include "page32.h"
+
 typedef struct _spinlock {
  int val;
 } spinlock;
 
-#define INIT_SPIN_LOCK(lock) memset(lock, 0, sizeof(spinlock))
+#define INIT_SPIN_LOCK(lock) memset((lock), 0, sizeof(spinlock))
 
 void spinlock_lock(spinlock *lock);
 void spinlock_unlock(spinlock *lock);

page32.c

@@ -11,21 +11,31 @@
 
 #include "page32.h"
 #include "system.h"
+#include "lock.h"
 
-extern addr_t _kernel_pg_dir;
 extern addr_t _kernel_pg_table_0;
+extern spinlock spinlock_smp;
+extern int smp_bits;
+extern int smp_on_hold;
 
-void build_pagetable(pgd_t *pgdir, pte_t **pgtable, u32 phys_addr, u32 start, u32 length, u32 pgd_flags, u32 pte_flags) {
+spinlock lock_pgd[(PAGE_SIZE-KERNEL_PGDIR_OFFSET)/sizeof(pgd_t)];
+
+void build_pagetable(u32 cpuid, pte_t **pgtable, u32 phys_addr, u32 start, u32 length, u32 pgd_flags, u32 pte_flags) {
 
  int idx = 0;
+ pgd_t *pgdir = GET_CPU_PGDIR(cpuid);
  pgd_t *pgd = pgdir + ((start >> PGD_SHIFT) & 0x3ff);
  pte_t *pte = pgtable[idx] + ((start >> PAGE_SHIFT) & 0x3ff);
  pte_t *last_pte = (pte_t*)(((long)pte + PAGE_SIZE) & PAGE_MASK);
  u32 end = start + length;
 
+ spinlock_lock(&lock_pgd[pgd-pgdir]);
  if (!*pgd) {
  *pgd = ((long)pgtable[idx] - KERNEL_VIRT_ADDR) | pgd_flags;
+ } else if (((*pgd & PGD_FLAG_MASK) ^ pgd_flags) != 0) {
+ *pgd |= pgd_flags;
  }
+ spinlock_unlock(&lock_pgd[pgd-pgdir]);
 
  while(start < end) {
 
@@ -36,11 +46,13 @@ void build_pagetable(pgd_t *pgdir, pte_t **pgtable, u32 phys_addr, u32 start, u3
  last_pte = pte + (PAGE_SIZE/sizeof(pte_t));
  pgd++;
 
+ spinlock_lock(&lock_pgd[pgd-pgdir]);
  if (!*pgd) {
  *pgd = (((long)pgtable[idx] - KERNEL_VIRT_ADDR) | pgd_flags);
- } else if ((*pgd & PAGE_MASK) != pgd_flags) {
+ } else if (((*pgd & PGD_FLAG_MASK) ^ pgd_flags) != 0) {
  *pgd |= pgd_flags;
  }
+ spinlock_unlock(&lock_pgd[pgd-pgdir]);
  }
 
  *pte = (phys_addr | pte_flags);
@@ -55,6 +67,9 @@ void map_kernel_linear_with_pagetable(addr_t virt_addr, u32 length, u32 pte_flag
  pte_t *pgtbl[1];
  pte_t *pte;
  addr_t end_virt_addr;
+ int i;
+ u32 local_smp_bits;
+ bool first_core;
 
  end_virt_addr = ALIGN_PAGE(virt_addr + length);
  virt_addr = virt_addr & PAGE_MASK;
@@ -63,19 +78,49 @@ void map_kernel_linear_with_pagetable(addr_t virt_addr, u32 length, u32 pte_flag
  pte += (((long)(virt_addr) >> PAGE_SHIFT) & 0x3ff);
 
  while (virt_addr != end_virt_addr) {
+ first_core = false;
+ spinlock_lock(&spinlock_smp);
+ local_smp_bits = smp_bits;
+ if (virt_addr != end_virt_addr)
+ smp_on_hold++;
+ spinlock_unlock(&spinlock_smp);
  pgtbl[0] = (pte_t*)((long)pte & PAGE_MASK);
- build_pagetable(GET_CURCPU_PGDIR, pgtbl, virt_addr - KERNEL_VIRT_ADDR, virt_addr, PAGE_SIZE, PGD_PRESENT | PGD_WRITE, pte_flags);
+ for (i = 0; i < MAX_NUM_SMPS; i++) {
+ if ((1 << i) & local_smp_bits) {
+ build_pagetable(i, pgtbl, virt_addr - KERNEL_VIRT_ADDR, virt_addr, PAGE_SIZE, PGD_PRESENT | PGD_WRITE, pte_flags);
+ if (!first_core) {
+ first_core = true;
+ spinlock_lock(&spinlock_smp);
+ smp_on_hold--;
+ spinlock_unlock(&spinlock_smp);
+ }
+ }
+ }
  virt_addr = ADDPTRS(virt_addr, PAGE_SIZE);
  pte++;
  }
 }
 
-void map_kernel_linear(pte_t **pgtable, addr_t virt_addr, u32 length, u32 pte_flags) {
- build_pagetable(GET_CURCPU_PGDIR, pgtable, virt_addr - KERNEL_VIRT_ADDR, virt_addr, length, PGD_PRESENT | PGD_WRITE, pte_flags);
-}
-
 pfn_t page_getpfn(void *addr) {
  pgd_t *pgd = GET_CURCPU_PGDIR + (((long)(addr) >> PGD_SHIFT) & 0x3ff);
  pte_t *pte = ((pte_t*)ADDPTRS(*pgd, KERNEL_VIRT_ADDR)) + (((long)(addr) >> PAGE_SHIFT) & 0x3ff);
  return (*pte & PAGE_MASK);
 }
+
+void pgd_lock_init() {
+ u32 i;
+ for(i = 0; i < (sizeof(lock_pgd)/sizeof(spinlock)); i++)
+ INIT_SPIN_LOCK(&lock_pgd[i]);
+}
+
+void pgd_lock_all() {
+ u32 i;
+ for(i=0;i<sizeof(lock_pgd)/sizeof(spinlock);i++)
+ spinlock_lock(&lock_pgd[i]);
+}
+
+void pgd_unlock_all() {
+ u32 i;
+ for(i=0;i<sizeof(lock_pgd)/sizeof(spinlock);i++)
+ spinlock_unlock(&lock_pgd[i]);
+}

page32.h

@@ -19,7 +19,12 @@ typedef u32 pte_t;
 typedef u32 pgd_t;
 typedef u32 pfn_t;
 
+extern addr_t _kernel_pg_dir;
+
+#define KERNEL_PGDIR_ENTRY KERNEL_PGDIR_OFFSET/sizeof(pgd_t)
+
 #define GET_CURCPU_PGDIR ((pgd_t*)((int)&_kernel_pg_dir + (PAGE_SIZE*CUR_CPU)))
+#define GET_CPU_PGDIR(cpuid) ((pgd_t*)((int)&_kernel_pg_dir + (PAGE_SIZE*(cpuid))))
 
 #define PGD_PRESENT 1
 #define PGD_WRITE 2
@@ -31,13 +36,16 @@ typedef u32 pfn_t;
 
 #define PAGE_MASK ~(PAGE_SIZE-1)
 #define PGD_MASK ~(PGD_SIZE-1)
+#define PGD_FLAG_MASK 0x1FFF
 
 #define ALIGN_PGD(m) (((m)+PGD_SIZE-1)&PGD_MASK)
 #define ALIGN_PAGE(m) (((m)+PAGE_SIZE-1)&PAGE_MASK)
 
-void build_pagetable(pgd_t *pgdir, pte_t **pgtable, u32 phys_addr, u32 start, u32 length, u32 pgd_flags, u32 pte_flags);
-void map_kernel_linear(pte_t **pgtable, addr_t virt_addr, u32 length, u32 pte_flags);
+void build_pagetable(u32 cpuid, pte_t **pgtable, u32 phys_addr, u32 start, u32 length, u32 pgd_flags, u32 pte_flags);
 pfn_t page_getpfn(void *addr);
 void map_kernel_linear_with_pagetable(addr_t virt_addr, u32 length, u32 pte_flags);
+void pgd_lock_init();
+void pgd_lock_all();
+void pgd_unlock_all();
 
 #endif

smp.c

@@ -24,24 +24,39 @@ extern int init_ap;
 extern int init_ap_size;
 
 extern int lapic_base_register;
-
-extern int _kernel_pg_dir;
 extern int _kernel_pg_table_0;
-
 extern int _kernel_stack_0_start;
 
-int smp_bits = 0;
+int smp_bits = 1;
 int smp_nums = 0;
+int smp_on_hold = 0;
+bool smp_init_waiting;
 
 spinlock spinlock_smp;
 
 void finish_smp_initialization(int smp_id) {
+ pgd_t *pgd;
+ int i;
 
+retry:
  spinlock_lock(&spinlock_smp);
+ if (smp_on_hold) {
+ spinlock_unlock(&spinlock_smp);
+ goto retry;
+ }
 
  smp_nums++;
  smp_bits |= 1 << smp_id;
 
+ if (!smp_init_waiting) {
+ pgd_lock_all();
+ for(i = 1; i < MAX_NUM_SMPS; i++) {
+ pgd = GET_CURCPU_PGDIR;
+ memcpy(pgd, &_kernel_pg_dir, PAGE_SIZE);
+ }
+ pgd_unlock_all();
+ }
+
  spinlock_unlock(&spinlock_smp);
 
  loadGDT32();
@@ -57,7 +72,6 @@ void finish_smp_initialization(int smp_id) {
 
  // clear identity mapping
  GET_CURCPU_PGDIR[0] = 0;
- //((int*)((int)&_kernel_pg_dir+(CUR_CPU*PAGE_SIZE)))[0] = 0;
  __asm__ __volatile__("invlpg (0);"
  : : :
  );
@@ -86,14 +100,12 @@ void initialize_kernel_pg_tables() {
  pgd = (pgd_t*)((int)&_kernel_pg_dir + (PAGE_SIZE * i));
  memcpy(pgd, &_kernel_pg_dir, PAGE_SIZE);
  // identity mapping for first 4 MB
- *pgd = *(pgd+(KERNEL_PGDIR_ENTRY/4));
+ *pgd = *(pgd+KERNEL_PGDIR_ENTRY);
  }
 }
 
 void smp_start() {
 
- INIT_SPIN_LOCK(&spinlock_smp);
-
  if (!lapic_present)
  return;
 
@@ -110,6 +122,8 @@ void smp_start() {
 
  MFENCE;
 
+ smp_init_waiting = true;
+
  // send INIT IPI to APs
  lapic_write_register(LAPIC_ICR_1, 0x000c4500);
  lapic_write_register(LAPIC_ICR_2, 0);
@@ -120,6 +134,10 @@ void smp_start() {
  lapic_write_register(LAPIC_ICR_2, 0);
  pit_wait_ms(200);
 
+ spinlock_lock(&spinlock_smp);
+ smp_init_waiting = false;
+ spinlock_unlock(&spinlock_smp);
+
  printf(KERNEL_INFO, "Number of APs: %d\n", smp_nums);
  printf(KERNEL_INFO, "SMP bits: %x\n", smp_bits);
 }

start.c

@@ -20,14 +20,19 @@
 #include "acpi.h"
 #include "debug.h"
 #include "interrupt.h"
+#include "lock.h"
 
 //
 // Start kernel routine
 //
 
+extern spinlock spinlock_smp;
+
 int start_kernel(void) {
 
  vga_init();
+ INIT_SPIN_LOCK(&spinlock_smp);
+ pgd_lock_init();
  if(mem_init())
  return -1;
  debug_init();

usermode.c

@@ -22,8 +22,6 @@ char __attribute__((aligned(4096))) um_pg_table[MAX_NUM_SMPS][PAGE_SIZE];
 #define USER_MODE_STACK_SIZE 8192
 char __attribute__((aligned(8))) user_mode_stack[MAX_NUM_SMPS][USER_MODE_STACK_SIZE];
 
-extern int _kernel_pg_dir;
-
 extern int init_um;
 extern int init_um_size;
 
@@ -59,14 +57,14 @@ void initialize_usermode() {
 
  pgtable[0] = (pte_t*)&um_pg_table[CUR_CPU][0];
 
- build_pagetable((pgd_t*)((int)&_kernel_pg_dir + (PAGE_SIZE*CUR_CPU)),
+ build_pagetable(CUR_CPU,
  pgtable,
  (int)&init_um-KERNEL_VIRT_ADDR,
  USER_MODE_VIRT_TEXT,
  (int)init_um_size,
  PGD_PRESENT | PGD_WRITE | PGD_USER, PTE_PRESENT | PTE_WRITE | PTE_USER);
 
- build_pagetable((pgd_t*)((int)&_kernel_pg_dir + (PAGE_SIZE*CUR_CPU)),
+ build_pagetable(CUR_CPU,
  pgtable,
  (int)&user_mode_stack[CUR_CPU][0]-KERNEL_VIRT_ADDR,
  USER_MODE_VIRT_STACK,