diff --git a/src/frontends/qt4/GuiRef.cpp b/src/frontends/qt4/GuiRef.cpp
index 555ccce1a4..795736c728 100644
--- a/src/frontends/qt4/GuiRef.cpp
+++ b/src/frontends/qt4/GuiRef.cpp
@@ -118,9 +118,12 @@ void GuiRef::gotoClicked()
 	// to which we are going (or from which we are returning) is
 	// restored in the dialog. It's a bit of a hack, but it works,
 	// and no-one seems to have any better idea.
-	last_reference_ = referenceED->text();
+	bool const toggled = last_reference_.empty();
+	if (toggled)
+		last_reference_ = referenceED->text();
 	gotoRef();
-	last_reference_.clear();
+	if (toggled)
+		last_reference_.clear();
 }