WIP: Simpler memlayout

src/arch/ppc64/bootblock_crt0.S

@@ -26,6 +26,56 @@
 	oris    r,r, (e)@h;			\
 	ori     r,r, (e)@l;
 
+/* Load an immediate 32-bit value into a register */
+#define LOAD_IMM32(r, e)			\
+	li      r, 0;				\
+	oris    r,r, (e)@h;			\
+	ori     r,r, (e)@l;
+
+/*
+ * On POWER, 0 is wherever HRMOR points to rather than physical DRAM start.
+ * HRMOR is ORed with address, not added to it, meaning that memory space
+ * overlaps after 2^(least significant set bit of HRMOR). This becomes
+ * chaotic when nonconsecutive bits are set...
+ *
+ * Two and a half possible cases:
+ * 0. bootblock started with QEMU in hb-mode
+ *    - NIA =                          0x10  (bug?)
+ *    - HRMOR =                  0x08000000  (128M)
+ *    - no physical memory to enable/train, everything accessible from start
+ * 1. bootblock loaded by HBBL
+ *    - NIA =                             0
+ *    - HRMOR =                  0xF8000000  (4G - 128M)
+ *    - initialized L3 =           0x400000  (4M)
+ *    - top address before RAM = 0xF8400000
+ * 2. bootblock in SEEPROM, loaded by SBE
+ *    - NIA =                        0x3000  (placeholder for int. vectors)
+ *    - HRMOR =                  0xF8200000  (4G - 128 M + 2 M)
+ *    - initialized L3 =             0x8000  (bootblock/HBBL size = 32K)
+ *    - no way 32K will be enough, must initialize more L3 in bootblock
+ *    - HRMOR still applies, so memory overlaps every 2M
+ *
+ * Common subset (assuming 2. initializes as much memory as possible) is
+ * 0xF8200000-0xF8400000. 2M should be more than enough for pre-RAM code,
+ * but it isn't enough to load ramstage. We could implement postcar stage,
+ * but KISS: initialize L3 from _ebootblock to 0xF8980000: up to 9.5M into
+ * cache, leaving bottom 2M (0xF8000000-0xF8200000) either uninitialized
+ * (when started from SEEPROM) or just unused for anything but bootblock
+ * (loaded by HBBL). Last 0.5M of L3 cache is left for interrupt vectors
+ * normally located at address 0.
+ *
+ * Set HRMOR to 0 before jumping to C code in bootblock and forget it even
+ * exists.
+ *
+ * For QEMU s/0xF8/0x08/ in above description but code remains the same.
+ * L3 initialization is unnecessary in this case but won't break anything.
+ *
+ * TODO: there is a structure with SBE->HBBL data at 0 in 2nd option. It
+ * holds some useful data like XSCOM BAR and LPC BAR. If, for any reason,
+ * these addresses are different than default, they should be used instead
+ * of predefined values.
+ */
+
 .section ".text._start", "ax", %progbits
 .globl _start
 _start:
@@ -39,90 +89,64 @@ _start:
 #endif
 
 	/* Set program priority to medium */
-	or		%r2, %r2, %r2
-
-#if CONFIG(BOOTBLOCK_IN_SEEPROM)
-	/* SBE sets HRMOR to 4G - 128M + 2M. Our bootblock is loaded 12 KB above
-	 * that address - it is used as a place for interrupt vectors in Hostboot,
-	 * but it isn't actually a part of the HBBL partition to save space in
-	 * SEEPROM. At the entry we are not loaded at the address we are linked for.
-	 *
-	 * HRMOR isn't added to the addresses, it is OR'ed with them. Because the
-	 * bit corresponding to 2M is set, address space overlaps every 2M. To be
-	 * able to use full 10M of L3 cache we must change it to something with
-	 * bigger alignment.
-	 */
+	or	%r2, %r2, %r2
 
-	lis		%r12, 0x20		/* Bit for 2M */
 	mfspr	%r11, 313		/* HRMOR */
-	li		%r10, 1
-	rotldi	%r10, %r10, 63	/* Most significant bit is "ignore HRMOR" */
+	li	%r10, 1
+	rotldi	%r10, %r10, 63		/* MSB is "ignore HRMOR" */
 
-	/* Bit for 2M is set both in ignoreHRMOR and HRMOR, that is expected */
-	lis		%r9, ignoreHRMOR@h
-	ori		%r9, %r9, ignoreHRMOR@l
-	or		%r9, %r9, %r11
-	or		%r9, %r9, %r10
+	/* Assumption: we are linked at address that isn't changed by HRMOR */
+	LOAD_IMM32(%r7, ignoreHRMOR)
+	or	%r9, %r7, %r10
 
 	mtlr %r9
 	blr
 
 ignoreHRMOR:
-	/* Now we are at address 0x80000000F82030xx */
-	andc	%r11, %r11, %r12	/* Clear bit for 2M */
-	mtspr	SPR_HRMOR, %r11		/* Update HRMOR */
+	/* Now we are at 0x8000000000000000 | linked address */
+	li	%r0, 0
+	mtspr	SPR_HRMOR, %r0		/* Clear HRMOR */
 	isync
 
 	/* We can't just "b stopIgnoringHRMOR", it would use relative offset */
-	addi	%r9, %r9, stopIgnoringHRMOR - ignoreHRMOR
-	mtlr %r9
+	addi	%r9, %r7, stopIgnoringHRMOR - ignoreHRMOR
+	mtlr	%r9
 	blr
 
 stopIgnoringHRMOR:
 	/* Now we are at linked address */
+	slbia 7
+	sync
+	isync
 
-	/* L3 cache is invalid except for [2M, end of HBBL] range. Make the rest of
-	 * it valid, or embrace the checkstops. End of loaded part of HBBL is at
-	 * address 2M + 12 (see https://github.com/open-power/hostboot/blob/master/src/import/chips/p9/procedures/hwp/nest/p9_sbe_hb_structures.H#L84),
-	 * the first 4 bytes are for the jump into HBBL). */
+	/*
+	 * When coming from SBE, L3 cache is invalid except for [2M, end of HBBL]
+	 * range. Make the rest of it valid, or embrace the checkstops.
+	 */
 
-	/* Validate and initialize to zeroes the first 2M */
-	li		%r5, 0
-	lis		%r6, 0x20
+	/* Validate and initialize to zeroes [end of HBBL, 9.5M] range */
+	LOAD_IMM32(%r5, _ebootblock)	/* Assume it is at least 128B aligned */
+	LOAD_IMM32(%r6, _epreram_cbfs_cache)	/* Same */
 	addi	%r6, %r6, -1
 1:
 	dcbz	0, %r5
 	addi	%r5, %r5, 128
 	cmpld	cr7, %r5, %r6
-	blt		cr7, 1b
-
-	/* Validate and initialize to zeroes [end of HBBL, 10M] range */
-	ld	%r5, 12(%r12)
-	add	%r5, %r5, %r12
-	addis	%r6, %r6, 0x80		/* Add 8M = 10M - 2M */
-1:
-	dcbz	0, %r5
-	addi	%r5, %r5, 128
-	cmpld	cr7, %r5, %r6
-	blt		cr7, 1b
-
-#endif
+	blt	cr7, 1b
 
 	/* Stack */
-	lis		%r1, _estack@ha
-	addi	%r1, %r1, _estack@l
+	LOAD_IMM32(%r1, _estack)
 
 	/* This is tested by checkstack() just before jumping to payload */
 	LOAD_IMM64(%r3, 0xDEADBEEFDEADBEEF)
-	lis		%r5, _stack@ha
-	addi	%r5, %r5, _stack@l
+	LOAD_IMM32(%r5, _stack)
 	subi	%r5, %r5, 8
-	sub		%r4, %r1, %r5
+	sub	%r4, %r1, %r5
 	sradi	%r4, %r4, 3		/* Divide by 8 */
 	mtctr	%r4
 1:
 	stdu	%r3, 8(%r5)
-	bc		25, 0, 1b
+	bc	25, 0, 1b
 
 	/* Enable floating point and vector operations */
 	/* Vector operations are sometimes generated for code like
@@ -131,13 +155,12 @@ stopIgnoringHRMOR:
 	 * there is no reason not to. Floating point must also be enabled for VSX.
 	 */
 	mfmsr	%r3
-	ori		%r3, %r3, 0x2000	/* FP = 1 */
+	ori	%r3, %r3, 0x2000	/* FP = 1 */
 	oris	%r3, %r3, 0x0280	/* VEC = 1, VSX = 1 */
-	mtmsr	%r3
+	mtmsrd	%r3
 
 	/* Load official procedure descriptor address for main() */
-	lis		%r12, main@ha
-	addi	%r12, %r12, main@l
+	LOAD_IMM32(%r12, main)
 
 	/* Load TOC pointer and jump to main() */
 	ld		%r2, 8(%r12)

src/arch/ppc64/stages.c

@@ -20,29 +20,10 @@
 
 void stage_entry(uintptr_t stage_arg)
 {
-	if (!ENV_ROMSTAGE_OR_BEFORE) {
-		/*
-		 * This works if:
-		 * - this stage haven't use stack so far
-		 * - this function (stage_entry) doesn't use a stack - preparing a stack
-		 *   frame for main() doesn't count
-		 * - main() doesn't take arguments on stack
-		 * - main() doesn't return
-		 * - stage is linked at proper address that includes HRMOR so we don't
-		 *   have to play with "ignore HRMOR" bit
-		 *
-		 * Also, 'sync' is for changing HRMOR, not stack.
-		 */
-		register uint64_t hrmor = read_spr(SPR_HRMOR);
-		asm volatile("sync; isync" ::: "memory");
-		write_spr(SPR_HRMOR, 0);
-		asm volatile("or 1,1,%0; slbia 7; sync; isync" :: "r"(hrmor) : "memory");
-
+	if (!ENV_ROMSTAGE_OR_BEFORE)
 		_cbmem_top_ptr = stage_arg;
-	}
-	else {
+	else
 		timestamp_init(read_spr(SPR_TB));
-	}
 
 	main();
 }

src/mainboard/raptor-cs/talos-2/memlayout.ld

@@ -6,40 +6,73 @@
 
 SECTIONS
 {
-	// On POWER, 0 is wherever HRMOR points to rather than physical DRAM start.
-	// HRMOR is ORed with address, not added to it.
 	DRAM_START(0x0)
 
+	/*
+	 * On POWER, 0 is wherever HRMOR points to rather than physical DRAM start.
+	 * HRMOR is ORed with address, not added to it, meaning that memory space
+	 * overlaps after 2^(least significant set bit of HRMOR). This becomes
+	 * chaotic when nonconsecutive bits are set...
+	 *
+	 * Two and a half possible cases:
+	 * 0. bootblock started with QEMU in hb-mode
+	 *    - NIA =                          0x10  (bug?)
+	 *    - HRMOR =                  0x08000000  (128M)
+	 *    - no physical memory to enable/train, everything accessible from start
+	 * 1. bootblock loaded by HBBL
+	 *    - NIA =                             0
+	 *    - HRMOR =                  0xF8000000  (4G - 128M)
+	 *    - initialized L3 =           0x400000  (4M)
+	 *    - top address before RAM = 0xF8400000
+	 * 2. bootblock in SEEPROM, loaded by SBE
+	 *    - NIA =                        0x3000  (placeholder for int. vectors)
+	 *    - HRMOR =                  0xF8200000  (4G - 128 M + 2 M)
+	 *    - initialized L3 =             0x8000  (bootblock/HBBL size = 32K)
+	 *    - no way 32K will be enough, must initialize more L3 in bootblock
+	 *    - HRMOR still applies, so memory overlaps every 2M
+	 *
+	 * Common subset (assuming 2. initializes as much memory as possible) is
+	 * 0xF8200000-0xF8400000. 2M should be more than enough for pre-RAM code,
+	 * but it isn't enough to load ramstage. We could implement postcar stage,
+	 * but KISS: initialize L3 from _ebootblock to 0xF8980000: up to 9.5M into
+	 * cache, leaving bottom 2M (0xF8000000-0xF8200000) either uninitialized
+	 * (when started from SEEPROM) or just unused for anything but bootblock
+	 * (loaded by HBBL). Last 0.5M of L3 cache is left for interrupt vectors
+	 * normally located at address 0.
+	 *
+	 * Set HRMOR to 0 before jumping to C code in bootblock and forget it even
+	 * exists.
+	 *
+	 * For QEMU s/0xF8/0x08/ in above description but code remains the same.
+	 * L3 initialization is unnecessary in this case but won't break anything.
+	 *
+	 * TODO: there is a structure with SBE->HBBL data at 0 in 2nd option. It
+	 * holds some useful data like XSCOM BAR and LPC BAR. If, for any reason,
+	 * these addresses are different than default, they should be used instead
+	 * of predefined values.
+	 */
+
 #if !CONFIG(BOOTBLOCK_IN_SEEPROM)
-	BOOTBLOCK(0, 32K)
+	BOOTBLOCK(           0xF8000000, 32K)
+#else
+	BOOTBLOCK(           0xF8203000, 20K)
 #endif
 
-#if !ENV_RAMSTAGE
-	STACK(0x8000, 32K)
-	PRERAM_CBMEM_CONSOLE(0x10000, 128K)
-	FMAP_CACHE(0x30000, 4K)
-	CBFS_MCACHE(0x31000, 8K)
-	TIMESTAMP(0x33000, 4K)
-	CBFS_CACHE(0x34000, 512K)
-#endif
+	STACK(               0xF8208000, 32K)
+	PRERAM_CBMEM_CONSOLE(0xF8210000, 128K)
+	FMAP_CACHE(          0xF8230000, 4K)
+	CBFS_MCACHE(         0xF8231000, 8K)
+	TIMESTAMP(           0xF8233000, 4K)
 
-	ROMSTAGE(0x100000, 1M)
+	ROMSTAGE(            0xF8240000, 256K)
 
-#if CONFIG(BOOTBLOCK_IN_SEEPROM)
-	BOOTBLOCK(0x203000, 20K)
-#endif
+	/*
+	 * bootblock_crt0.S assumes this is the last part of L3, leaving for
+	 * interrupt vectors at least 0.5M because of cache associativity. If
+	 * more CBFS_CACHE is needed, split this into pre-/postram caches.
+	 */
+	CBFS_CACHE(          0xF8280000, 7M)
 
-#if !ENV_RAMSTAGE
-	RAMSTAGE(0x300000, 5M)
-#else
-
-	STACK(0xf8008000, 32K)
+	RAMSTAGE(            0xF9000000, 2M)
 
-	PRERAM_CBMEM_CONSOLE(0xf8010000, 128K)
-	FMAP_CACHE(0xf8030000, 4K)
-	CBFS_MCACHE(0xf8031000, 8K)
-	TIMESTAMP(0xf8033000, 4K)
-	RAMSTAGE(0xf8300000, 5M)
-	CBFS_CACHE(0xf9000000, 20M)
-#endif
 }

src/soc/ibm/power9/timer.c

@@ -58,6 +58,15 @@ void init_timer(void)
 {
 	uint64_t tmp;
 
+	/*
+	 * 1. Data Cache Block set to Zero for 0..0x1000
+	 * 2. Instruction Cache Block Invalidate for 0..0x1000 - single 'isync'
+	 *    before enabling interrupts by writing to MSR is enough (not defined
+	 *    by ISA but in POWER9 Processor User's Manual, 4.6.2.2)
+	 */
+	for (tmp = 0; tmp < 0x1000; tmp += 128)
+		asm volatile("dcbz 0, %0; icbi 0, %0;" :: "r"(tmp) : "memory");
+
 	/*
 	 * Set both decrementers to the highest possible value. POWER9 implements
 	 * 56 bits, they decrement with 512MHz frequency. Decrementer exception
@@ -103,11 +112,12 @@ void init_timer(void)
 
 	*(uint32_t *)0x900 = 0x48000000;	// Decrementer
 
+	asm volatile("sync; isync" ::: "memory");
+
 	tmp = read_msr();
 	write_msr(tmp | 0x8000);	/* EE - External Interrupt Enable */
 }
 
-/* TODO: with HDEC we can get ~2ns resolution, may be useful for RAM init. */
 void udelay(unsigned int usec)
 {
 	uint64_t start = read_spr(SPR_TB);