diff --git a/src/main/java/de/bwravencl/controllerbuddy/input/Profile.java b/src/main/java/de/bwravencl/controllerbuddy/input/Profile.java
index e357e6ab..741f5ae9 100644
--- a/src/main/java/de/bwravencl/controllerbuddy/input/Profile.java
+++ b/src/main/java/de/bwravencl/controllerbuddy/input/Profile.java
@@ -25,6 +25,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.UUID;
 import org.lwjgl.glfw.GLFW;
 
@@ -100,6 +101,10 @@ public long getKeyRepeatInterval() {
 		return keyRepeatInterval;
 	}
 
+	public Optional<Mode> getModeByUuid(final UUID modeUuid) {
+		return modes.stream().filter(mode -> mode.getUuid().equals(modeUuid)).findFirst();
+	}
+
 	public List<Mode> getModes() {
 		return modes;
 	}
@@ -161,9 +166,8 @@ void setActiveMode(final Input input, final int index) {
 		}
 	}
 
-	public void setActiveMode(final Input input, final UUID modeUuid) {
-		modes.stream().filter(mode -> mode.getUuid().equals(modeUuid)).findFirst()
-				.ifPresent(mode -> setActiveMode(input, modes.indexOf(mode)));
+	public void setActiveMode(final Input input, final Mode mode) {
+		setActiveMode(input, modes.indexOf(mode));
 	}
 
 	private void setButtonToModeActionsMap(final Map<Integer, List<ButtonToModeAction>> buttonToModeActionMap) {
diff --git a/src/main/java/de/bwravencl/controllerbuddy/input/action/ButtonToModeAction.java b/src/main/java/de/bwravencl/controllerbuddy/input/action/ButtonToModeAction.java
index cee7e71c..2a05d9a3 100644
--- a/src/main/java/de/bwravencl/controllerbuddy/input/action/ButtonToModeAction.java
+++ b/src/main/java/de/bwravencl/controllerbuddy/input/action/ButtonToModeAction.java
@@ -57,7 +57,15 @@ public static List<ButtonToModeAction> getButtonToModeActionStack() {
 	private void activateMode(final Input input, final Profile profile) {
 		if (!buttonToModeActionStack.contains(this)) {
 			buttonToModeActionStack.push(this);
-			profile.setActiveMode(input, modeUuid);
+			final var activeMode = profile.getActiveMode();
+
+			profile.getModeByUuid(modeUuid).ifPresent(newMode -> {
+				IAxisToLongPressAction.onModeActivated(activeMode, newMode);
+				// noinspection UnnecessarilyQualifiedStaticUsage
+				IButtonToAction.onModeActivated(activeMode, newMode);
+
+				profile.setActiveMode(input, newMode);
+			});
 
 			if (targetsOnScreenKeyboardMode()) {
 				input.getMain().setOnScreenKeyboardVisible(true);
@@ -131,7 +139,11 @@ private void deactivateMode(final Input input, final Profile profile) {
 					.stream().filter(action -> action instanceof IAxisToAction).forEach(_ -> input.suspendAxis(axis)));
 		}
 
-		profile.setActiveMode(input, previousMode.getUuid());
+		IAxisToLongPressAction.onModeDeactivated(activeMode);
+		// noinspection UnnecessarilyQualifiedStaticUsage
+		IButtonToAction.onModeDeactivated(activeMode);
+
+		profile.setActiveMode(input, previousMode);
 
 		if (targetsOnScreenKeyboardMode()) {
 			main.setOnScreenKeyboardVisible(false);
diff --git a/src/main/java/de/bwravencl/controllerbuddy/input/action/IAxisToLongPressAction.java b/src/main/java/de/bwravencl/controllerbuddy/input/action/IAxisToLongPressAction.java
index 75e12724..d577da27 100644
--- a/src/main/java/de/bwravencl/controllerbuddy/input/action/IAxisToLongPressAction.java
+++ b/src/main/java/de/bwravencl/controllerbuddy/input/action/IAxisToLongPressAction.java
@@ -17,6 +17,7 @@
 package de.bwravencl.controllerbuddy.input.action;
 
 import de.bwravencl.controllerbuddy.input.Input;
+import de.bwravencl.controllerbuddy.input.Mode;
 import de.bwravencl.controllerbuddy.input.action.IActivatableAction.Activatable;
 import de.bwravencl.controllerbuddy.input.action.IActivatableAction.Activation;
 import java.util.HashMap;
@@ -31,6 +32,22 @@ private static boolean isOnReleaseAction(final IActivatableAction<?> action) {
 		return action.getActivation() == Activation.SINGLE_ON_RELEASE;
 	}
 
+	static void onModeActivated(final Mode activeMode, final Mode newMode) {
+		actionToDownSinceMap.keySet().removeIf(action -> {
+			if (activeMode.getAllActions().contains(action)) {
+				final var optionalAxisId = activeMode.getAxisToActionsMap().entrySet().stream()
+						.filter(entry -> entry.getValue().contains(action)).map(Map.Entry::getKey).findFirst();
+				return optionalAxisId.isPresent() && newMode.getButtonToActionsMap().containsKey(optionalAxisId.get());
+			}
+
+			return false;
+		});
+	}
+
+	static void onModeDeactivated(final Mode activeMode) {
+		actionToDownSinceMap.keySet().removeIf(action -> activeMode.getAllActions().contains(action));
+	}
+
 	static void reset() {
 		actionToDownSinceMap.clear();
 		actionToMustDenyActivationMap.clear();
diff --git a/src/main/java/de/bwravencl/controllerbuddy/input/action/IButtonToAction.java b/src/main/java/de/bwravencl/controllerbuddy/input/action/IButtonToAction.java
index 4e34754a..020d31aa 100644
--- a/src/main/java/de/bwravencl/controllerbuddy/input/action/IButtonToAction.java
+++ b/src/main/java/de/bwravencl/controllerbuddy/input/action/IButtonToAction.java
@@ -17,6 +17,7 @@
 package de.bwravencl.controllerbuddy.input.action;
 
 import de.bwravencl.controllerbuddy.input.Input;
+import de.bwravencl.controllerbuddy.input.Mode;
 import de.bwravencl.controllerbuddy.input.action.IActivatableAction.Activatable;
 import de.bwravencl.controllerbuddy.input.action.IActivatableAction.Activation;
 import java.util.HashMap;
@@ -31,6 +32,23 @@ private static boolean isOnReleaseAction(final IActivatableAction<?> action) {
 		return action.getActivation() == Activation.SINGLE_ON_RELEASE;
 	}
 
+	static void onModeActivated(final Mode activeMode, final Mode newMode) {
+		actionToDownSinceMap.keySet().removeIf(action -> {
+			if (activeMode.getAllActions().contains(action)) {
+				final var optionalButtonId = activeMode.getButtonToActionsMap().entrySet().stream()
+						.filter(entry -> entry.getValue().contains(action)).map(Map.Entry::getKey).findFirst();
+				return optionalButtonId.isPresent()
+						&& newMode.getButtonToActionsMap().containsKey(optionalButtonId.get());
+			}
+
+			return false;
+		});
+	}
+
+	static void onModeDeactivated(final Mode activeMode) {
+		actionToDownSinceMap.keySet().removeIf(action -> activeMode.getAllActions().contains(action));
+	}
+
 	static void reset() {
 		actionToDownSinceMap.clear();
 		actionToMustDenyActivationMap.clear();