Merge remote-tracking branch 'origin/freebsd/11-stable/master' into h…

…ardened/11-stable/master

Signed-off-by: Oliver Pinter <[email protected]>

sys/arm/arm/cpuinfo.c

@@ -31,6 +31,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/pcpu.h>
+#include <sys/smp.h>
 #include <sys/sysctl.h>
 
 #include <machine/cpu.h>
@@ -40,6 +42,9 @@ __FBSDID("$FreeBSD$");
 
 #if __ARM_ARCH >= 6
 void reinit_mmu(uint32_t ttb, uint32_t aux_clr, uint32_t aux_set);
+
+int disable_bp_hardening;
+int spectre_v2_safe = 1;
 #endif
 
 struct cpuinfo cpuinfo =
@@ -253,6 +258,7 @@ cpuinfo_get_actlr_modifier(uint32_t *actlr_mask, uint32_t *actlr_set)
 
 	if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) {
 		switch (cpuinfo.part_number) {
+		case CPU_ARCH_CORTEX_A75:
 		case CPU_ARCH_CORTEX_A73:
 		case CPU_ARCH_CORTEX_A72:
 		case CPU_ARCH_CORTEX_A57:
@@ -336,4 +342,197 @@ cpuinfo_reinit_mmu(uint32_t ttb)
 	reinit_mmu(ttb, actlr_mask, actlr_set);
 }
 
+static bool
+modify_actlr(uint32_t clear, uint32_t set)
+{
+	uint32_t reg, newreg;
+
+	reg = cp15_actlr_get();
+	newreg = reg;
+	newreg &= ~clear;
+	newreg |= set;
+	if (reg == newreg)
+		return (true);
+	cp15_actlr_set(newreg);
+
+	reg = cp15_actlr_get();
+	if (reg == newreg)
+		return (true);
+	return (false);
+}
+
+/* Apply/restore BP hardening on current core. */
+static int
+apply_bp_hardening(bool enable, int kind, bool actrl, uint32_t set_mask)
+{
+	if (enable) {
+		if (actrl && !modify_actlr(0, set_mask))
+			return (-1);
+		PCPU_SET(bp_harden_kind, kind);
+	} else {
+		PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE);
+		if (actrl)
+			modify_actlr(~0, PCPU_GET(original_actlr));
+		spectre_v2_safe = 0;
+	}
+	return (0);
+}
+
+static void
+handle_bp_hardening(bool enable)
+{
+	int kind;
+	char *kind_str;
+
+	kind = PCPU_BP_HARDEN_KIND_NONE;
+	/*
+	 * Note: Access to ACTRL is locked to secure world on most boards.
+	 * This means that full BP hardening depends on updated u-boot/firmware
+	 * or is impossible at all (if secure monitor is in on-chip ROM).
+	 */
+	if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) {
+		switch (cpuinfo.part_number) {
+		case CPU_ARCH_CORTEX_A8:
+			/*
+			 * For Cortex-A8, IBE bit must be set otherwise
+			 * BPIALL is effectively NOP.
+			 * Unfortunately, Cortex-A is also affected by
+			 * ARM erratum 687067 which causes non-working
+			 * BPIALL if IBE bit is set and 'Instruction L1 System
+			 * Array Debug Register 0' is not 0.
+			 * This register is not reset on power-up and is
+			 * accessible only from secure world, so we cannot do
+			 * nothing (nor detect) to fix this issue.
+			 * I afraid that on chip ROM based secure monitor on
+			 * AM335x (BeagleBone) doesn't reset this debug
+			 * register.
+			 */
+			kind = PCPU_BP_HARDEN_KIND_BPIALL;
+			if (apply_bp_hardening(enable, kind, true, 1 << 6) != 0)
+				goto actlr_err;
+			break;
+		break;
+
+		case CPU_ARCH_CORTEX_A9:
+		case CPU_ARCH_CORTEX_A12:
+		case CPU_ARCH_CORTEX_A17:
+		case CPU_ARCH_CORTEX_A57:
+		case CPU_ARCH_CORTEX_A72:
+		case CPU_ARCH_CORTEX_A73:
+		case CPU_ARCH_CORTEX_A75:
+			kind = PCPU_BP_HARDEN_KIND_BPIALL;
+			if (apply_bp_hardening(enable, kind, false, 0) != 0)
+				goto actlr_err;
+			break;
+
+		case CPU_ARCH_CORTEX_A15:
+			/*
+			 * For Cortex-A15, set 'Enable invalidates of BTB' bit.
+			 * Despite this, the BPIALL is still effectively NOP,
+			 * but with this bit set, the ICIALLU also flushes
+			 * branch predictor as side effect.
+			 */
+			kind = PCPU_BP_HARDEN_KIND_ICIALLU;
+			if (apply_bp_hardening(enable, kind, true, 1 << 0) != 0)
+				goto actlr_err;
+			break;
+
+		default:
+			break;
+		}
+	} else if (cpuinfo.implementer == CPU_IMPLEMENTER_QCOM) {
+		printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative "
+		    "branch attacks. !!!\n"
+		    "Qualcomm Krait cores are known (or believed) to be "
+		    "vulnerable to \n"
+		    "speculative branch attacks, no mitigation exists yet.\n",
+		    PCPU_GET(cpuid));
+		goto unkonown_mitigation;
+	}  else {
+		goto unkonown_mitigation;
+	}
+
+	if (bootverbose) {
+		switch (kind) {
+		case PCPU_BP_HARDEN_KIND_NONE:
+			kind_str = "not necessary";
+			break;
+		case PCPU_BP_HARDEN_KIND_BPIALL:
+			kind_str = "BPIALL";
+			break;
+		case PCPU_BP_HARDEN_KIND_ICIALLU:
+			kind_str = "ICIALLU";
+			break;
+		default:
+			panic("Unknown BP hardering kind (%d).", kind);
+		}
+		printf("CPU(%d) applied BP hardening: %s\n", PCPU_GET(cpuid),
+		    kind_str);
+	}
+
+	return;
+
+unkonown_mitigation:
+	PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE);
+	spectre_v2_safe = 0;
+	return;
+
+actlr_err:
+	PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE);
+	spectre_v2_safe = 0;
+	printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative branch "
+	    "attacks. !!!\n"
+	    "We cannot enable required bit(s) in ACTRL register\n"
+	    "because it's locked by secure monitor and/or firmware.\n",
+	    PCPU_GET(cpuid));
+}
+
+void
+cpuinfo_init_bp_hardening(void)
+{
+
+	/*
+	 * Store original unmodified ACTRL, so we can restore it when
+	 * BP hardening is disabled by sysctl.
+	 */
+	PCPU_SET(original_actlr, cp15_actlr_get());
+	handle_bp_hardening(true);
+}
+
+static void
+bp_hardening_action(void *arg)
+{
+
+	handle_bp_hardening(disable_bp_hardening == 0);
+}
+
+static int
+sysctl_disable_bp_hardening(SYSCTL_HANDLER_ARGS)
+{
+	int rv;
+
+	rv = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
+
+	if (!rv && req->newptr) {
+		spectre_v2_safe = 1;
+		dmb();
+#ifdef SMP
+		smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier,
+		bp_hardening_action, NULL, NULL);
+#else
+		bp_hardening_action(NULL);
+#endif
+	}
+
+	return (rv);
+}
+
+SYSCTL_PROC(_machdep, OID_AUTO, disable_bp_hardening,
+    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
+    &disable_bp_hardening, 0, sysctl_disable_bp_hardening, "I",
+    "Disable BP hardening mitigation.");
+
+SYSCTL_INT(_machdep, OID_AUTO, spectre_v2_safe, CTLFLAG_RD,
+    &spectre_v2_safe, 0, "System is safe to Spectre Version 2 attacks");
+
 #endif /* __ARM_ARCH >= 6 */

sys/arm/arm/genassym.c

@@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/cpuset.h>
 #include <sys/systm.h>
 #include <sys/assym.h>
+#include <sys/pcpu.h>
 #include <sys/proc.h>
 #include <sys/mbuf.h>
 #include <sys/vmmeter.h>
@@ -134,6 +135,10 @@ ASSYM(PCB_VFPSTATE, offsetof(struct pcb, pcb_vfpstate));
 
 #if __ARM_ARCH >= 6
 ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap));
+ASSYM(PC_BP_HARDEN_KIND, offsetof(struct pcpu, pc_bp_harden_kind));
+ASSYM(PCPU_BP_HARDEN_KIND_NONE, PCPU_BP_HARDEN_KIND_NONE);
+ASSYM(PCPU_BP_HARDEN_KIND_BPIALL, PCPU_BP_HARDEN_KIND_BPIALL);
+ASSYM(PCPU_BP_HARDEN_KIND_ICIALLU, PCPU_BP_HARDEN_KIND_ICIALLU);
 #endif
 
 ASSYM(PAGE_SIZE, PAGE_SIZE);

sys/arm/arm/machdep.c

@@ -1262,6 +1262,8 @@ initarm(struct arm_boot_params *abp)
 	msgbufinit(msgbufp, msgbufsize);
 	dbg_monitor_init();
 	arm_kdb_init();
+	/* Apply possible BP hardening. */
+	cpuinfo_init_bp_hardening();
 	return ((void *)STACKALIGN(thread0.td_pcb));
 
 }

sys/arm/arm/mp_machdep.c

@@ -200,6 +200,9 @@ init_secondary(int cpu)
 	/* Configure the interrupt controller */
 	intr_pic_init_secondary();
 
+	/* Apply possible BP hardening */
+	cpuinfo_init_bp_hardening();
+
 	mtx_lock_spin(&ap_boot_mtx);
 
 	atomic_add_rel_32(&smp_cpus, 1);

sys/arm/arm/swtch-v6.S

@@ -145,7 +145,16 @@ ENTRY(cpu_context_switch)
 	* predictors and Requirements for branch predictor maintenance
 	* operations sections.
 	*/
-	mcr	CP15_BPIALL		/* flush entire Branch Target Cache */
+	/*
+	 * Additionally, to mitigate mistrained branch predictor attack
+	 * we must invalidate it on affected CPUs. Unfortunately, BPIALL
+	 * is effectively NOP on Cortex-A15 so it needs special treatment.
+	 */
+	ldr	r0, [r8, #PC_BP_HARDEN_KIND]
+	cmp	r0, #PCPU_BP_HARDEN_KIND_ICIALLU
+	mcrne	CP15_BPIALL		/* Flush entire Branch Target Cache   */
+	mcreq	CP15_ICIALLU		/* This is the only way how to flush  */
+					/* Branch Target Cache on Cortex-A15. */
 	DSB
 	mov	pc, lr
 END(cpu_context_switch)

sys/arm/arm/trap-v6.c

@@ -287,6 +287,7 @@ abort_handler(struct trapframe *tf, int prefetch)
 	struct vmspace *vm;
 	vm_prot_t ftype;
 	bool usermode;
+	int bp_harden;
 #ifdef INVARIANTS
 	void *onfault;
 #endif
@@ -303,6 +304,20 @@ abort_handler(struct trapframe *tf, int prefetch)
 
 	idx = FSR_TO_FAULT(fsr);
 	usermode = TRAPF_USERMODE(tf);	/* Abort came from user mode? */
+
+	/*
+	 * Apply BP hardening by flushing the branch prediction cache
+	 * for prefaults on kernel addresses.
+	 */
+	if (__predict_false(prefetch && far > VM_MAXUSER_ADDRESS &&
+	    (idx == FAULT_TRAN_L2 || idx == FAULT_PERM_L2))) {
+		bp_harden = PCPU_GET(bp_harden_kind);
+		if (bp_harden == PCPU_BP_HARDEN_KIND_BPIALL)
+			_CP15_BPIALL();
+		else if (bp_harden == PCPU_BP_HARDEN_KIND_ICIALLU)
+			_CP15_ICIALLU();
+	}
+
 	if (usermode)
 		td->td_frame = tf;
 

sys/arm/include/cpuinfo.h

@@ -49,6 +49,7 @@
 #define CPU_ARCH_CORTEX_A57		0xD07
 #define CPU_ARCH_CORTEX_A72		0xD08
 #define CPU_ARCH_CORTEX_A73		0xD09
+#define CPU_ARCH_CORTEX_A75		0xD0A
 
 
 /* QCOM */
@@ -122,6 +123,7 @@ extern struct cpuinfo cpuinfo;
 
 void cpuinfo_init(void);
 #if __ARM_ARCH >= 6
+void cpuinfo_init_bp_hardening(void);
 void cpuinfo_reinit_mmu(uint32_t ttb);
 #endif
 #endif	/* _MACHINE_CPUINFO_H_ */

sys/arm/include/pcpu.h

@@ -42,6 +42,10 @@ struct vmspace;
 #endif	/* _KERNEL */
 
 #if __ARM_ARCH >= 6
+/* Branch predictor hardening method */
+#define PCPU_BP_HARDEN_KIND_NONE		0
+#define PCPU_BP_HARDEN_KIND_BPIALL		1
+#define PCPU_BP_HARDEN_KIND_ICIALLU		2
 
 #define PCPU_MD_FIELDS							\
 	unsigned int pc_vfpsid;						\
@@ -57,7 +61,9 @@ struct vmspace;
 	void *pc_qmap_pte2p;						\
 	unsigned int pc_dbreg[32];					\
 	int pc_dbreg_cmd;						\
-	char __pad[19]
+	int pc_bp_harden_kind;						\
+	uint32_t pc_original_actlr;					\
+	char __pad[11]
 #else
 #define PCPU_MD_FIELDS							\
 	char __pad[157]

sys/dev/vt/vt_font.c

@@ -42,6 +42,7 @@ static MALLOC_DEFINE(M_VTFONT, "vtfont", "vt font");
 
 /* Some limits to prevent abnormal fonts from being loaded. */
 #define	VTFONT_MAXMAPPINGS	65536
+#define	VTFONT_MAXGLYPHS	131072
 #define	VTFONT_MAXGLYPHSIZE	2097152
 #define	VTFONT_MAXDIMENSION	128
 
@@ -171,7 +172,8 @@ vtfont_load(vfnt_t *f, struct vt_font **ret)
 	/* Make sure the dimensions are valid. */
 	if (f->width < 1 || f->height < 1)
 		return (EINVAL);
-	if (f->width > VTFONT_MAXDIMENSION || f->height > VTFONT_MAXDIMENSION)
+	if (f->width > VTFONT_MAXDIMENSION || f->height > VTFONT_MAXDIMENSION ||
+	    f->glyph_count > VTFONT_MAXGLYPHS)
 		return (E2BIG);
 
 	/* Not too many mappings. */