Implement a better solution for painting of RTL text

Instead of relying on character range (Hebrew or Arabic) or character
direction, use RLO unicode character (Right-to-Left override) to force
painting in the direction indicated by the current font. This should
be as close as we can to the old LyX behavior (and requires less
code).

If this code works as intended, it will be possible to remove a lot of
code from Encodings.cpp.

00README_STR_METRICS_BRANCH

@@ -11,8 +11,7 @@ what we have with force_paint_single_char. Moreover there has been
 some code factorization in TextMetrics, where the same row-breaking
 algorithm was basically implemented 3 times.
 
-Currently everything is supposed to work for LTR text, and RtL text
-should work too except possibly metrics with Arabic and Hebrew fonts.
+Currently everything is supposed to work for both LTR and RTL text.
 
 When KEEP_OLD_METRICS_CODE is defined in TextMetrics.cpp, the new code
 is tested against the old one in getPosNearX and cursorX. This can be

src/Encoding.cpp

@@ -711,12 +711,6 @@ docstring Encodings::fromLaTeXCommand(docstring const & cmd, int cmdtype,
 }
 
 
-bool Encodings::isHebrewChar(char_type c)
-{
-	return c >= 0x0590 && c <= 0x05ff;
-}
-
-
 bool Encodings::isHebrewComposeChar(char_type c)
 {
 	return c <= 0x05c2 && c >= 0x05b0 && c != 0x05be && c != 0x05c0;

src/Encoding.h

@@ -269,8 +269,6 @@ public:
 	///
 	static bool isHebrewComposeChar(char_type c);
 	///
-	static bool isHebrewChar(char_type c);
-	///
 	static bool isArabicComposeChar(char_type c);
 	///
 	static bool isArabicSpecialChar(char_type c);

src/rowpainter.cpp

@@ -20,7 +20,6 @@
 #include "BufferParams.h"
 #include "BufferView.h"
 #include "Changes.h"
-#include "Encoding.h"
 #include "Language.h"
 #include "Layout.h"
 #include "LyXRC.h"
@@ -201,12 +200,6 @@ void RowPainter::paintChars(pos_type & vpos, Font const & font)
 	bool const spell_state =
 		lyxrc.spellcheck_continuously && par_.isMisspelled(pos);
 
-	// are we building a RtL string? 
-	//FIXME: I would like to use the new isRTL() from textutils.h,
-	// but it does not give the same results for some reason I do
-	// not understand.
-	bool const rtl = Encodings::isArabicChar(str[0]) || Encodings::isHebrewChar(str[0]);
-
 	// collect as much similar chars as we can
 	for (++vpos ; vpos < end ; ++vpos) {
 		if (lyxrc.force_paint_single_char)
@@ -234,14 +227,6 @@ void RowPainter::paintChars(pos_type & vpos, Font const & font)
 
 		char_type c = par_.getChar(pos);
 
-		//FIXME: I would like to use the new isRTL() from textutils.h,
-		// but it does not give the same results for some reason I do
-		// not understand.
-		bool const new_rtl = Encodings::isArabicChar(c) || Encodings::isHebrewChar(c);
-		if (new_rtl != rtl)
-			// String direction has changed
-			break;
-
 		if (c == '\t')
 			break;
 
@@ -263,20 +248,29 @@ void RowPainter::paintChars(pos_type & vpos, Font const & font)
 
 	docstring s(&str[0], str.size());
 
+	if (s[0] == '\t')
+		s.replace(0,1,from_ascii("    "));
+
 	/* Because we do our own bidi, at this point the strings are
 	 * already in visual order. However, Qt also applies its own
 	 * bidi algorithm to strings that it paints to the screen.
 	 * Therefore, if we were to paint Hebrew/Arabic words as a
 	 * single string, the letters in the words would get reversed
-	 * again. In order to avoid that, we reverse the string in advance.
+	 * again. In order to avoid that, we force LTR drawing.
 	 * See also http://thread.gmane.org/gmane.editors.lyx.devel/79740
 	 * for an earlier thread on the subject
 	 */
-	if (rtl)
+	// Left-to-right override: forces to draw text left-to-right
+	char_type const LRO = 0x202D;
+	// Right-to-left override: forces to draw text right-to-left
+	char_type const RLO = 0x202E;
+	// Pop directional formatting: return to previous state
+	char_type const PDF = 0x202C;
+	if (font.isVisibleRightToLeft()) {
 		reverse(s.begin(), s.end());
-
-	if (s[0] == '\t')
-		s.replace(0,1,from_ascii("    "));
+		s = RLO + s + PDF;
+	} else
+		s = LRO + s + PDF;
 
 	if (!selection && !change_running.changed()) {
 		x_ += pi_.pain.text(int(x_), yo_, s, font.fontInfo());

src/support/lstrings.cpp

@@ -161,29 +161,6 @@ bool isNumber(char_type c)
 }
 
 
-bool isRTL(char_type c)
-{
-	if (!is_utf16(c))
-		// assume that no non-utf16 character is right-to-left
-		// c outside the UCS4 range is catched as well
-		return false;
-	QChar::Direction direction = ucs4_to_qchar(c).direction();
-	/**
-	 * See for example:
-	 *  http://en.wikipedia.org/wiki/Template:Bidi_Class_%28Unicode%29.
-	 * Here we only handle the easy cases:
-	 *  * R:  Hebrew alphabet and related punctuation
-	 *  * AL: Arabic, Thaana and Syriac alphabets, and most
-	 *        punctuation specific to those scripts
-	 *
-	 * FIXME: testing show that this does not work (see
-	 * RowPainter::paintChars), but my knowledge of unicode is too
-	 * poor to understand why.
-	 */
-	return direction == QChar::DirR || direction ==QChar::DirAL;
-}
-
-
 bool isDigitASCII(char_type c)
 {
 	return '0' <= c && c <= '9';

src/support/textutils.h

@@ -44,9 +44,6 @@ bool isSpace(char_type c);
 /// return true if a unicode char is a numeral.
 bool isNumber(char_type c);
 
-/// return true is a unicode char uses a right-to-left direction for layout
-bool isRTL(char_type c);
-
 /// return whether \p c is a digit in the ASCII range
 bool isDigitASCII(char_type c);