fix: make block syscall in asm

Author: joway

poll_default_linux.go

@@ -99,9 +99,7 @@ func (p *defaultPoll) Wait() (err error) {
 		}
 		// msec: 0(raw) => 1ms(sched,raw) => -1(block_syscall)
 		// poller's G will hold P at most 1ms
-		if msec > 0 {
-			n, err = EpollWaitRaw(p.fd, p.events, msec)
-		} else if msec == 0 {
+		if msec >= 0 {
 			n, err = EpollWaitRaw(p.fd, p.events, msec)
 		} else { // < 0
 			n, err = EpollWaitBlock(p.fd, p.events, msec)

sys_epoll_linux.go

@@ -28,6 +28,16 @@ func entersyscallblock()
 //go:linkname exitsyscall runtime.exitsyscall
 func exitsyscall()
 
+//go:nosplit
+func callEntersyscallblock() {
+	entersyscallblock()
+}
+
+//go:nosplit
+func callExitsyscall() {
+	exitsyscall()
+}
+
 // EpollCreate implements epoll_create1.
 func EpollCreate(flag int) (fd int, err error) {
 	var r0 uintptr
@@ -68,18 +78,13 @@ func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) {
 
 func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) {
 	r0, _, errno := BlockSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0)
-	if errno == syscall.Errno(0) {
+	if errno == 0 {
 		err = nil
 	} else {
-		err = errno
+		err = syscall.Errno(errno)
 	}
 	return int(r0), err
 }
 
-//go:nosplit
-func BlockSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) {
-	entersyscallblock()
-	r1, r2, err = syscall.RawSyscall6(trap, a1, a2, a3, a4, a5, a6)
-	exitsyscall()
-	return r1, r2, err
-}
+//go:noescape
+func BlockSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr)

sys_epoll_linux_amd64.s

@@ -0,0 +1,41 @@
+// Copyright 2024 CloudWeGo Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "textflag.h"
+
+// func BlockSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr)
+TEXT ·BlockSyscall6<ABIInternal>(SB),NOSPLIT,$0
+	CALL	·callEntersyscallblock(SB)
+	// a6 already in R9.
+	// a5 already in R8.
+	MOVQ	SI, R10 // a4
+	MOVQ	DI, DX  // a3
+	MOVQ	CX, SI  // a2
+	MOVQ	BX, DI  // a1
+	// num already in AX.
+	SYSCALL
+	CMPQ	AX, $0xfffffffffffff001
+	JLS	ok
+	NEGQ	AX
+	MOVQ	AX, CX  // errno
+	MOVQ	$-1, AX // r1
+	MOVQ	$0, BX  // r2
+	CALL	·callExitsyscall(SB)
+	RET
+ok:
+	// r1 already in AX.
+	MOVQ	DX, BX // r2
+	MOVQ	$0, CX // errno
+	CALL	·callExitsyscall(SB)
+	RET

sys_epoll_linux_arm64.s

@@ -0,0 +1,42 @@
+// Copyright 2024 CloudWeGo Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "textflag.h"
+
+// func BlockSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr)
+TEXT ·BlockSyscall6(SB),NOSPLIT,$0-80
+	BL	·callEntersyscallblock(SB)
+	MOVD	num+0(FP), R8	// syscall entry
+	MOVD	a1+8(FP), R0
+	MOVD	a2+16(FP), R1
+	MOVD	a3+24(FP), R2
+	MOVD	a4+32(FP), R3
+	MOVD	a5+40(FP), R4
+	MOVD	a6+48(FP), R5
+	SVC
+	CMN	$4095, R0
+	BCC	ok
+	MOVD	$-1, R4
+	MOVD	R4, r1+56(FP)
+	MOVD	ZR, r2+64(FP)
+	NEG	R0, R0
+	MOVD	R0, errno+72(FP)
+	BL	·callExitsyscall(SB)
+	RET
+ok:
+	MOVD	R0, r1+56(FP)
+	MOVD	R1, r2+64(FP)
+	MOVD	ZR, errno+72(FP)
+	BL	·callExitsyscall(SB)
+	RET