memset

arch/armv7a/memset.S

@@ -34,76 +34,113 @@ memset:
 	str r0, [sp, #-8]!
 	mov DST, r0
 
-	/* is dst aligned to 8? */
-	tst DST, #7
-	beq .Laligned8
+	and VAL_H, #0xff
+	orr VAL_H, VAL_H, VAL_H, lsl #8
+	orr VAL_H, VAL_H, VAL_H, lsl #16
+
+	/* quickly deal with < 64 byte len */
+	cmp LEN, #64
+	bhs .Lmemset64
+
+.Ltail63:
+	/* TMP = LEN / 4 */
+	movs TMP, LEN, lsr #2
+	beq .Ltail63_0
+
+.Ltail63_4:
+	str VAL_H, [DST], #4
+	subs TMP, TMP, #1
+	bne .Ltail63_4
+
+.Ltail63_0:
+	/* LEN = LEN % 4 */
+	ands LEN, #3
+	beq .Lreturn
 
-	/* align to 8 */
 1:
 	strb VAL_H, [DST], #1
-	sub LEN, LEN, #1
-	cbz LEN, .Lreturn  /* return if len == 0 */
-	tst DST, #7
+	subs LEN, LEN, #1
 	bne 1b
 
-.Laligned8:
-	and VAL_H, #0xff
-	orr VAL_H, VAL_H, VAL_H, lsl #8
-	orr VAL_H, VAL_H, VAL_H, lsl #16
+.Lreturn:
+	ldr r0, [sp], #8
+	bx lr
 
-	/* do we have more to store than 8? */
-	cmp LEN, #8
-	bhs .Lmemset8
+.Lmemset64:
+	/* at least 64 bytes to store */
 
-	/* less than 8 bytes to store, can't do it with dwords */
-	tst LEN, #4
-	it ne
-	strne VAL_H, [DST], #4
+	/* aligned to 64? */
+	ands TMP, DST, #63
+	beq .Laligned64
 
-	tst LEN, #2
+	rsb TMP, #64
+	sub LEN, TMP
+	cmp TMP, #4
+	blo 3f
+
+	sub TMP, #4
+2:
+	/* store 4 bytes at a time */
+	str VAL_H, [DST], #4
+	subs TMP, TMP, #4
+	bhs 2b
+
+3:
+	/* 1-3 bytes to store */
+	tst TMP, #2
 	itt ne
 	strbne VAL_H, [DST], #1
 	strbne VAL_H, [DST], #1
 
-	tst LEN, #1
+	tst TMP, #1
 	it ne
 	strbne VAL_H, [DST], #1
 
-	b .Lreturn
+	/* < 64 after aligning? */
+	cmp LEN, #64
+	blo .Ltail63
+
+.Laligned64:
+	/* at least 64 bytes to store, we're 64-byte aligned */
+	mov VAL_L, VAL_H
+
+	/* store 64 bytes at a time */
+	sub LEN, LEN, #64
+	sub DST, #8
+
+4:
+	strd VAL_L, VAL_H, [DST, #8]
+	strd VAL_L, VAL_H, [DST, #16]
+	strd VAL_L, VAL_H, [DST, #24]
+	strd VAL_L, VAL_H, [DST, #32]
+	strd VAL_L, VAL_H, [DST, #40]
+	strd VAL_L, VAL_H, [DST, #48]
+	strd VAL_L, VAL_H, [DST, #56]
+	strd VAL_L, VAL_H, [DST, #64]!
+	subs LEN, LEN, #64
+	bhs 4b
+
+	add DST, #8
+
+	ands LEN, LEN, #63
+	beq .Lreturn  /* return if len == 0 */
+	cmp LEN, #8
+	blo .Ltail63
 
 .Lmemset8:
 	/* at least 8 bytes to store, we're 8-byte aligned */
-	mov VAL_L, VAL_H
 
 	/* store 8 bytes at a time */
 	sub LEN, #8
 
-2:
+5:
 	strd VAL_L, VAL_H, [DST], #8
 	subs LEN, LEN, #8
-	bhs 2b
+	bhs 5b
 
 	/* make LEN positive again */
 	ands LEN, #7
 	beq .Lreturn
 
-	/* store remaining bytes */
-	tst LEN, #4
-	it ne
-	strne VAL_H, [DST], #4
-
-	tst LEN, #2
-	itt ne
-	strbne VAL_H, [DST], #1
-	strbne VAL_H, [DST], #1
-
-	tst LEN, #1
-	it ne
-	strbne VAL_H, [DST], #1
-
-
-.Lreturn:
-	ldr r0, [sp], #8
-	bx lr
-
+	b .Ltail63
 .size memset, .-memset