diff --git a/src/BufferView.cpp b/src/BufferView.cpp
index 1b2752e638..ec77a81c46 100644
--- a/src/BufferView.cpp
+++ b/src/BufferView.cpp
@@ -3197,6 +3197,7 @@ void BufferView::updateMetrics(bool force)
 		d->text_metrics_.clear();
 	}
 
+	// This should not be moved earlier
 	TextMetrics & tm = textMetrics(&buftext);
 
 	// make sure inline completion pointer is ok
@@ -3212,14 +3213,16 @@ void BufferView::updateMetrics(bool force)
 
 	// Check that the end of the document is not too high
 	int const min_visible = lyxrc.scroll_below_document ? minVisiblePart() : height_;
-	if (tm.last().first == lastpit && tm.last().second->bottom() < min_visible) {
+	if (tm.last().first == lastpit && tm.last().second->hasPosition()
+	     && tm.last().second->bottom() < min_visible) {
 		d->anchor_ypos_ += min_visible - tm.last().second->bottom();
 		LYXERR(Debug::SCROLLING, "Too high, adjusting anchor ypos to " << d->anchor_ypos_);
 		tm.updateMetrics(d->anchor_pit_, d->anchor_ypos_, height_);
 	}
 
 	// Check that the start of the document is not too low
-	if (tm.first().first == 0 && tm.first().second->top() > 0) {
+	if (tm.first().first == 0 && tm.first().second->hasPosition()
+	     && tm.first().second->top() > 0) {
 		d->anchor_ypos_ -= tm.first().second->top();
 		LYXERR(Debug::SCROLLING, "Too low, adjusting anchor ypos to " << d->anchor_ypos_);
 		tm.updateMetrics(d->anchor_pit_, d->anchor_ypos_, height_);
@@ -3232,11 +3235,11 @@ void BufferView::updateMetrics(bool force)
 	 * extra paragraphs are removed
 	 */
 	// Remove paragraphs that are outside of screen
-	while(tm.first().second->bottom() <= 0) {
+	while(!tm.first().second->hasPosition() || tm.first().second->bottom() <= 0) {
 		//LYXERR0("Forget pit: " << tm.first().first);
 		tm.forget(tm.first().first);
 	}
-	while(tm.last().second->top() > height_) {
+	while(!tm.last().second->hasPosition() || tm.last().second->top() > height_) {
 		//LYXERR0("Forget pit: " << tm.first().first);
 		tm.forget(tm.last().first);
 	}
diff --git a/src/ParagraphMetrics.cpp b/src/ParagraphMetrics.cpp
index 31b31a2d01..5bf7cb1d43 100644
--- a/src/ParagraphMetrics.cpp
+++ b/src/ParagraphMetrics.cpp
@@ -40,9 +40,10 @@ using namespace lyx::support;
 
 namespace lyx {
 
+const int pm_npos = -10000;
 
 ParagraphMetrics::ParagraphMetrics(Paragraph const & par) :
-	position_(-1), id_(par.id()), par_(&par)
+	position_(pm_npos), id_(par.id()), par_(&par)
 {}
 
 
@@ -61,7 +62,14 @@ void ParagraphMetrics::reset(Paragraph const & par)
 {
 	par_ = &par;
 	dim_ = Dimension();
-	//position_ = -1;
+	//position_ = pm_npos;
+}
+
+
+int ParagraphMetrics::position() const
+{
+	LASSERT(hasPosition(), return -1);
+	return position_;
 }
 
 
@@ -71,6 +79,18 @@ void ParagraphMetrics::setPosition(int position)
 }
 
 
+void ParagraphMetrics::resetPosition()
+{
+	position_ = pm_npos;
+}
+
+
+bool ParagraphMetrics::hasPosition() const
+{
+	return position_ != pm_npos;
+}
+
+
 Row const & ParagraphMetrics::getRow(pos_type pos, bool boundary) const
 {
 	LBUFERR(!rows().empty());
diff --git a/src/ParagraphMetrics.h b/src/ParagraphMetrics.h
index 805d056541..b572f122b5 100644
--- a/src/ParagraphMetrics.h
+++ b/src/ParagraphMetrics.h
@@ -70,8 +70,12 @@ public:
 	bool hfillExpansion(Row const & row, pos_type pos) const;
 
 	/// The vertical position of the baseline of the first line of the paragraph
-	int position() const { return position_; }
+	int position() const;
 	void setPosition(int position);
+	/// Set position to unknown
+	void resetPosition();
+	/// Return true when the position of the paragraph is known
+	bool hasPosition() const;
 	/// The vertical position of the top of the paragraph
 	int top() const { return position_ - dim_.ascent(); }
 	/// The vertical position of the bottom of the paragraph
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index 9979909f28..57e944455f 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -220,7 +220,10 @@ void TextMetrics::updateMetrics(pit_type const anchor_pit, int const anchor_ypos
                                 int const bv_height)
 {
 	LASSERT(text_->isMainText(), return);
-	pit_type const npit = pit_type(text_->paragraphs().size());
+
+	// Forget existing positions
+	for (auto & pm_pair : par_metrics_)
+		pm_pair.second.resetPosition();
 
 	if (!contains(anchor_pit))
 		// Rebreak anchor paragraph.
@@ -246,6 +249,7 @@ void TextMetrics::updateMetrics(pit_type const anchor_pit, int const anchor_ypos
 	int y2 = anchor_ypos + anchor_pm.descent();
 	// We are now just below the anchor paragraph.
 	pit_type pit2 = anchor_pit + 1;
+	pit_type const npit = pit_type(text_->paragraphs().size());
 	for (; pit2 < npit && y2 < bv_height; ++pit2) {
 		if (!contains(pit2))
 			redoParagraph(pit2);