diff --git a/src/Cursor.cpp b/src/Cursor.cpp
index 225e4a685c..c67f38f178 100644
--- a/src/Cursor.cpp
+++ b/src/Cursor.cpp
@@ -1288,7 +1288,9 @@ bool Cursor::posVisToNewRow(bool movingLeft)
 	// if moving left in an LTR paragraph or moving right in an
 	// RTL one, move to previous row
 	if (par_is_LTR == movingLeft) {
-		if (row.pos() == 0) { // we're at first row in paragraph
+		if (row.start_boundary())
+			boundary(true);
+		else if (row.pos() == 0) { // we're at first row in paragraph
 			if (pit() == 0) // no previous paragraph! don't move
 				return false;
 			// move to last pos in previous par
diff --git a/src/Row.h b/src/Row.h
index babe11510f..a1852e240e 100644
--- a/src/Row.h
+++ b/src/Row.h
@@ -212,6 +212,10 @@ public:
 	///
 	pos_type endpos() const { return end_; }
 	///
+	void start_boundary(bool b) { start_boundary_ = b; }
+	///
+	bool start_boundary() const { return start_boundary_; }
+	///
 	void end_boundary(bool b) { end_boundary_ = b; }
 	///
 	bool end_boundary() const { return end_boundary_; }
@@ -373,6 +377,8 @@ private:
 	pos_type pos_ = 0;
 	/// one behind last pos covered by this row
 	pos_type end_ = 0;
+	// Is there a boundary at the start of the row (display inset...)
+	bool start_boundary_ = false;
 	// Is there a boundary at the end of the row (display inset...)
 	bool end_boundary_ = false;
 	// Shall the row be flushed when it is supposed to be justified?
diff --git a/src/RowFlags.h b/src/RowFlags.h
index ccc7f5b46c..6f754dffb4 100644
--- a/src/RowFlags.h
+++ b/src/RowFlags.h
@@ -53,6 +53,8 @@ enum RowFlags {
 	// (default is center)
 	AlignLeft = 1 << 11,
 	AlignRight = 1 << 12,
+	// Forbid boundary after this element
+	NoEndBoundary = 1 << 13,
 	// A display element breaks row at both ends
 	Display = FlushBefore | BreakBefore | BreakAfter,
 	// Flags that concern breaking after element
diff --git a/src/Text.cpp b/src/Text.cpp
index b4d5232c7d..fa00e8b6f3 100644
--- a/src/Text.cpp
+++ b/src/Text.cpp
@@ -3054,29 +3054,16 @@ bool Text::cursorBackward(Cursor & cur)
 	// Tell BufferView to test for FitCursor in any case!
 	cur.screenUpdateFlags(Update::FitCursor);
 
+	// if on right side of a row boundary (at row start), skip it,
+	// i.e. set boundary to true, i.e. go only logically left
+	if (!cur.boundary()
+	     && cur.textRow().pos() == cur.pos()
+	     && cur.textRow().start_boundary()) {
+		return setCursor(cur, cur.pit(), cur.pos(), true, true);
+	}
+
 	// not at paragraph start?
 	if (cur.pos() > 0) {
-		// if on right side of boundary (i.e. not at paragraph end, but line end)
-		// -> skip it, i.e. set boundary to true, i.e. go only logically left
-		// there are some exceptions to ignore this: lineseps, newlines, spaces
-#if 0
-		// some effectless debug code to see the values in the debugger
-		bool bound = cur.boundary();
-		int rowpos = cur.textRow().pos();
-		int pos = cur.pos();
-		bool sep = cur.paragraph().isSeparator(cur.pos() - 1);
-		bool newline = cur.paragraph().isNewline(cur.pos() - 1);
-		bool linesep = cur.paragraph().isLineSeparator(cur.pos() - 1);
-#endif
-		if (!cur.boundary() &&
-				cur.textRow().pos() == cur.pos() &&
-				!cur.paragraph().isLineSeparator(cur.pos() - 1) &&
-				!cur.paragraph().isNewline(cur.pos() - 1) &&
-				!cur.paragraph().isEnvSeparator(cur.pos() - 1) &&
-				!cur.paragraph().isSeparator(cur.pos() - 1)) {
-			return setCursor(cur, cur.pit(), cur.pos(), true, true);
-		}
-
 		// go left and try to enter inset
 		if (checkAndActivateInset(cur, false))
 			return false;
@@ -3143,33 +3130,14 @@ bool Text::cursorForward(Cursor & cur)
 
 		// next position is left of boundary,
 		// but go to next line for special cases like space, newline, linesep
-#if 0
-		// some effectless debug code to see the values in the debugger
-		int endpos = cur.textRow().endpos();
-		int lastpos = cur.lastpos();
-		int pos = cur.pos();
-		bool linesep = cur.paragraph().isLineSeparator(cur.pos());
-		bool newline = cur.paragraph().isNewline(cur.pos());
-		bool sep = cur.paragraph().isSeparator(cur.pos());
-		if (cur.pos() != cur.lastpos()) {
-			bool linesep2 = cur.paragraph().isLineSeparator(cur.pos()+1);
-			bool newline2 = cur.paragraph().isNewline(cur.pos()+1);
-			bool sep2 = cur.paragraph().isSeparator(cur.pos()+1);
-		}
-#endif
 		if (cur.textRow().endpos() == cur.pos() + 1) {
 			if (cur.paragraph().isEnvSeparator(cur.pos()) &&
 			    cur.pos() + 1 == cur.lastpos() &&
 			    cur.pit() != cur.lastpit()) {
 				// move to next paragraph
 				return setCursor(cur, cur.pit() + 1, 0, true, false);
-			} else if (cur.textRow().endpos() != cur.lastpos() &&
-				   !cur.paragraph().isNewline(cur.pos()) &&
-				   !cur.paragraph().isEnvSeparator(cur.pos()) &&
-				   !cur.paragraph().isLineSeparator(cur.pos()) &&
-				   !cur.paragraph().isSeparator(cur.pos())) {
+			} else if (cur.textRow().end_boundary())
 				return setCursor(cur, cur.pit(), cur.pos() + 1, true, true);
-			}
 		}
 
 		// in front of RTL boundary? Stay on this side of the boundary because:
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index 84e43c1d7d..3d98213da7 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -1147,7 +1147,9 @@ void cleanupRow(Row & row, bool at_end)
 	if (!at_end && !row.flushed())
 		row.back().rtrim();
 	// boundary exists when there was no space at the end of row
-	row.end_boundary(!at_end && row.back().endpos == row.endpos());
+	row.end_boundary(!at_end
+	                 && row.back().endpos == row.endpos()
+	                 && !(row.back().row_flags & NoEndBoundary));
 	// make sure that the RTL elements are in reverse ordering
 	row.reverseRTL();
 }
@@ -1250,6 +1252,13 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) const
 			rows.back().needsChangeBar(true);
 	}
 
+	// Set start_boundary to be equal to the previous row's end boundary
+	bool sb = false;
+	for (auto & row : rows) {
+		row.start_boundary(sb);
+		sb = row.end_boundary();
+	}
+
 	return rows;
 }
 
diff --git a/src/insets/InsetNewline.cpp b/src/insets/InsetNewline.cpp
index 38898c8ecd..8878fa95bf 100644
--- a/src/insets/InsetNewline.cpp
+++ b/src/insets/InsetNewline.cpp
@@ -44,9 +44,9 @@ InsetNewline::InsetNewline() : Inset(nullptr)
 int InsetNewline::rowFlags() const
 {
 	if (params_.kind == InsetNewlineParams::LINEBREAK)
-		return AlwaysBreakAfter;
+		return AlwaysBreakAfter | NoEndBoundary;
 	else
-	    return AlwaysBreakAfter | Flush;
+	    return AlwaysBreakAfter | NoEndBoundary | Flush;
 }
 
 
diff --git a/src/insets/InsetSeparator.h b/src/insets/InsetSeparator.h
index dba716c6aa..dbe549d626 100644
--- a/src/insets/InsetSeparator.h
+++ b/src/insets/InsetSeparator.h
@@ -65,7 +65,7 @@ public:
 		return docstring();
 	}
 	///
-	int rowFlags() const override { return BreakAfter | Flush; }
+	int rowFlags() const override { return BreakAfter | Flush | NoEndBoundary; }
 	///
 	bool nextnoindent() const { return params_.kind == InsetSeparatorParams::PLAIN; }
 private: