diff --git a/scene/gui/base_button.cpp b/scene/gui/base_button.cpp
index ed7e0de0e2..72299f788d 100644
--- a/scene/gui/base_button.cpp
+++ b/scene/gui/base_button.cpp
@@ -144,7 +144,9 @@ void BaseButton::_toggled(bool p_pressed) {
 }
 
 void BaseButton::on_action_event(Ref<InputEvent> p_event) {
-	if (p_event->is_pressed()) {
+	Ref<InputEventMouseButton> mouse_button = p_event;
+
+	if (p_event->is_pressed() && (mouse_button.is_null() || status.hovering)) {
 		status.press_attempt = true;
 		status.pressing_inside = true;
 		emit_signal(SNAME("button_down"));
@@ -174,12 +176,6 @@ void BaseButton::on_action_event(Ref<InputEvent> p_event) {
 	}
 
 	if (!p_event->is_pressed()) {
-		Ref<InputEventMouseButton> mouse_button = p_event;
-		if (mouse_button.is_valid()) {
-			if (!has_point(mouse_button->get_position())) {
-				status.hovering = false;
-			}
-		}
 		status.press_attempt = false;
 		status.pressing_inside = false;
 		emit_signal(SNAME("button_up"));
diff --git a/scene/main/viewport.cpp b/scene/main/viewport.cpp
index 44463ef063..a7ff213d65 100644
--- a/scene/main/viewport.cpp
+++ b/scene/main/viewport.cpp
@@ -1783,7 +1783,9 @@ void Viewport::_gui_input_event(Ref<InputEvent> p_event) {
 					Control *control = Object::cast_to<Control>(ci);
 					if (control) {
 						if (control->get_focus_mode() != Control::FOCUS_NONE) {
-							if (control != gui.key_focus) {
+							// Grabbing unhovered focus can cause issues when mouse is dragged
+							// with another button held down.
+							if (control != gui.key_focus && gui.mouse_over_hierarchy.has(control)) {
 								control->grab_focus();
 							}
 							break;