diff --git a/serial_unix.go b/serial_unix.go
index e6913c2..b7892e2 100644
--- a/serial_unix.go
+++ b/serial_unix.go
@@ -150,6 +150,9 @@ func (port *unixPort) SetMode(mode *Mode) error {
 		return err
 	} else if special {
 		requireSpecialBaudrate = true
+		// fall back to a standard speed for setTermSettings.
+		settings.Ispeed = unix.B9600
+		settings.Ospeed = unix.B9600
 	}
 	if err := port.setTermSettings(settings); err != nil {
 		return err
@@ -235,6 +238,13 @@ func nativeOpen(portName string, mode *Mode) (*unixPort, error) {
 		return nil, &PortError{code: InvalidSerialPort, causedBy: fmt.Errorf("error getting term settings: %w", err)}
 	}
 
+	if _, nonstandard := setTermSettingsBaudrate(mode.BaudRate, settings); nonstandard {
+		// the OS cached a non-standard baudrate. fall back to a standard speed for setTermSettings.
+		// the non-standard baudrate in mode.BaudRate will be set afterwards.
+		settings.Ispeed = unix.B9600
+		settings.Ospeed = unix.B9600
+	}
+
 	// Set raw mode
 	setRawMode(settings)