Correctly generate latex for font/language/encoding switches inside and around

insets. Up until now this has not been 100% correct, especially in the case
of RTL languages (but not only those).
This fixes bug 1820 (http://bugzilla.lyx.org/show_bug.cgi?id=1820)


git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@19150 a592a061-630c-0410-9148-cb99ea01b6c8

RELEASE-NOTES

@@ -35,6 +35,22 @@ When using multiple Windows to edit different parts of the
 same document, the cursor position is sometimes not correctly restored
 when you switch from one view to the other.
 
+- Languages/encodings and insets
+
+One of the bugs fixed in LyX 1.5.0 is that previously, there were certain 
+specific cases in which the LaTeX generated did not correctly reflect 
+language/encoding transitions in and around insets (footnotes, LyX notes).
+After much deliberation, it was decided not to change older files such that
+they will still reflect the old LaTeX output; rather, they will now correctly
+reflect the situation as it appears in the GUI. This means, however, that if 
+you mangled the text in the GUI in the older versions, in order that it
+generate the correct LaTeX output, the LaTeX will now generate the mangled 
+text. If this is problematic for you, please get in touch with us on the
+developers mailing list, we do have some possible solutions for this.
+
+The effects of this will be more pronounced for RTL (Hebrew, Arabic, Farsi) 
+users --- though they affect users of other languages as well.
+
 
 Note: There may later be an updated list of known issues online at
 	http://wiki.lyx.org/LyX/ReleaseNotes

src/Font.cpp

@@ -876,7 +876,8 @@ int Font::latexWriteStartChanges(odocstream & os, BufferParams const & bparams,
 int Font::latexWriteEndChanges(odocstream & os, BufferParams const & bparams,
 				  OutputParams const & runparams,
 				  Font const & base,
-				  Font const & next) const
+				  Font const & next,
+				  bool const & closeLanguage) const
 {
 	int count = 0;
 	bool env = false;
@@ -955,7 +956,8 @@ int Font::latexWriteEndChanges(odocstream & os, BufferParams const & bparams,
 		open_encoding_ = false;
 	}
 
-	if (language() != base.language() && language() != next.language()) {
+	if (closeLanguage &&
+			language() != base.language() && language() != next.language()) {
 		os << '}';
 		++count;
 	}

src/Font.h

@@ -309,7 +309,8 @@ public:
 	int latexWriteEndChanges(odocstream &, BufferParams const & bparams,
 				 OutputParams const & runparams,
 				 Font const & base,
-				 Font const & next) const;
+				 Font const & next,
+				 bool const & closeLanguage = true) const;
 
 
 	/// Build GUI description of font state

src/Paragraph.cpp

@@ -191,7 +191,7 @@ public:
 	///
 	void simpleTeXSpecialChars(Buffer const &, BufferParams const &,
 				   odocstream &,
-				   TexRow & texrow, OutputParams const &,
+				   TexRow & texrow, OutputParams &,
 				   Font & running_font,
 				   Font & basefont,
 				   Font const & outerfont,
@@ -662,7 +662,7 @@ void Paragraph::Pimpl::simpleTeXSpecialChars(Buffer const & buf,
 					     BufferParams const & bparams,
 					     odocstream & os,
 					     TexRow & texrow,
-					     OutputParams const & runparams,
+					     OutputParams & runparams,
 					     Font & running_font,
 					     Font & basefont,
 					     Font const & outerfont,
@@ -752,14 +752,35 @@ void Paragraph::Pimpl::simpleTeXSpecialChars(Buffer const & buf,
 // right now, which means stupid latex code like \textsf{}. AFAIK,
 // this does not harm dvi output. A minor bug, thus (JMarc)
 #endif
-		// some insets cannot be inside a font change command
+		// Some insets cannot be inside a font change command.
+		// However, even such insets *can* be placed in \L or \R
+		// or their equivalents (for RTL language switches), so we don't
+		// close the language in those cases.
+		// ArabTeX, though, cannot handle this special behavior, it seems.
+		bool arabtex = basefont.language()->lang() == "arabic_arabtex" ||
+					   running_font.language()->lang() == "arabic_arabtex";
 		if (open_font && inset->noFontChange()) {
-			column += running_font.latexWriteEndChanges(
+			bool closeLanguage = arabtex ||
+				basefont.isRightToLeft() == running_font.isRightToLeft();
+			unsigned int count = running_font.latexWriteEndChanges(
 					os, bparams, runparams,
-						basefont, basefont);
+						basefont, basefont, closeLanguage);
-			open_font = false;
+			column += count;
-			basefont = owner_->getLayoutFont(bparams, outerfont);
+			// if any font properties were closed, update the running_font, 
-			running_font = basefont;
+			// making sure, however, to leave the language as it was
+			if (count > 0) {
+				// FIXME: probably a better way to keep track of the old 
+				// language, than copying the entire font?
+				Font const copy_font(running_font);
+				basefont = owner_->getLayoutFont(bparams, outerfont);
+				running_font = basefont;
+				if (!closeLanguage)
+					running_font.setLanguage(copy_font.language());
+				// leave font open if language is still open
+				open_font = (running_font.language() == basefont.language());
+				if (closeLanguage)
+					runparams.local_font = &basefont;
+			}
 		}
 
 		int tmp = inset->latex(buf, os, runparams);
@@ -2120,6 +2141,10 @@ bool Paragraph::simpleTeXOnePar(Buffer const & buf,
 					texrow, rp, running_font,
 					basefont, outerfont, open_font,
 					runningChange, *style, i, column, c);
+
+		// Set the encoding to that returned from simpleTeXSpecialChars (see
+		// comment for encoding member in OutputParams.h)
+		runparams.encoding = rp.encoding;
 	}
 
 	// If we have an open font definition, we have to close it

src/output_latex.cpp

@@ -259,14 +259,23 @@ TeXOnePar(Buffer const & buf,
 	OutputParams runparams = runparams_in;
 	runparams.moving_arg |= style->needprotect;
 
+	// This paragraph's language
 	Language const * const par_language = pit->getParLanguage(bparams);
+	// The document's language
 	Language const * const doc_language = bparams.language;
-	Language const * const prev_par_language =
+	// The language that was in effect when the environemnt this paragraph is 
-		(pit != paragraphs.begin())
+	// inside of was opened
-		? boost::prior(pit)->getParLanguage(bparams)
+	Language const * const outer_language = 
-		: doc_language;
+		(runparams.local_font != 0) ?
+			runparams.local_font->language() : doc_language;
+	// The previous language that was in effect is either the language of
+	// the previous paragraph, if there is one, or else the outer language
+	// if there is no previous paragraph
+	Language const * const prev_language =
+		(pit != paragraphs.begin()) ?
+			boost::prior(pit)->getParLanguage(bparams) : outer_language;
 
-	if (par_language->babel() != prev_par_language->babel()
+	if (par_language->babel() != prev_language->babel()
 	    // check if we already put language command in TeXEnvironment()
 	    && !(style->isEnvironment()
 		 && (pit == paragraphs.begin() ||
@@ -275,19 +284,61 @@ TeXOnePar(Buffer const & buf,
 		     || boost::prior(pit)->getDepth() < pit->getDepth())))
 	{
 		if (!lyxrc.language_command_end.empty() &&
-		    prev_par_language->babel() != doc_language->babel() &&
+		    prev_language->babel() != outer_language->babel() &&
-		    !prev_par_language->babel().empty())
+		    !prev_language->babel().empty())
 		{
 			os << from_ascii(subst(lyxrc.language_command_end,
 				"$$lang",
-				prev_par_language->babel()))
+				prev_language->babel()))
 			   << '\n';
 			texrow.newline();
 		}
 
+		// We need to open a new language if we couldn't close the previous 
+		// one (because there's no language_command_end); and even if we closed
+		// the previous one, if the current language is different than the
+		// outer_language (which is currently in effect once the previous one
+		// is closed).
 		if ((lyxrc.language_command_end.empty() ||
-		     par_language->babel() != doc_language->babel()) &&
+		     par_language->babel() != outer_language->babel()) &&
 		    !par_language->babel().empty()) {
+			// If we're inside an inset, and that inset is within an \L or \R
+			// (or equivalents), then within the inset, too, any opposite
+			// language paragraph should appear within an \L or \R (in addition
+			// to, outside of, the normal language switch commands).
+			// This behavior is not correct for ArabTeX, though.
+			if (	// not for ArabTeX
+					(par_language->lang() != "arabic_arabtex" &&
+					 outer_language->lang() != "arabic_arabtex") &&
+					// are we in an inset?
+					runparams.local_font != 0 &&
+					// is the inset within an \L or \R?
+					// 
+					// FIXME: currently, we don't check this; this means that
+					// we'll have unnnecessary \L and \R commands, but that 
+					// doesn't seem to hurt (though latex will complain)
+					// 
+					// is this paragraph in the opposite direction?
+					runparams.local_font->isRightToLeft() !=
+						par_language->rightToLeft()
+				) {
+				// FIXME: I don't have a working copy of the Arabi package, so
+				// I'm not sure if the farsi and arabic_arabi stuff is correct
+				// or not...
+				if (par_language->lang() == "farsi")
+					os << "\\textFR{";
+				else if (outer_language->lang() == "farsi")
+					os << "\\textLR{";
+				else if (par_language->lang() == "arabic_arabi")
+					os << "\\textAR{";
+				else if (outer_language->lang() == "arabic_arabi")
+					os << "\\textLR{";
+				// remaining RTL languages currently is hebrew
+				else if (par_language->rightToLeft())
+					os << "\\R{";
+				else
+					os << "\\L{";
+			}
 			os << from_ascii(subst(
 				lyxrc.language_command_begin,
 				"$$lang",
@@ -457,8 +508,25 @@ TeXOnePar(Buffer const & buf,
 		}
 	}
 
-	if (boost::next(pit) == paragraphs.end()
+	// Closing the language is needed for the last paragraph; it is also
-	    && par_language->babel() != doc_language->babel()) {
+	// needed if we're within an \L or \R that we may have opened above (not
+	// necessarily in this paragraph) and are about to close.
+	bool closing_rtl_ltr_environment = 
+		// not for ArabTeX
+		(par_language->lang() != "arabic_arabtex" &&
+		 outer_language->lang() != "arabic_arabtex") &&
+		// have we opened and \L or \R environment?
+		runparams.local_font != 0 &&
+		runparams.local_font->isRightToLeft() != par_language->rightToLeft() &&
+		// are we about to close the language?
+		((boost::next(pit) != paragraphs.end() &&
+		  par_language->babel() != 
+		  	(boost::next(pit)->getParLanguage(bparams))->babel()) ||
+		 (boost::next(pit) == paragraphs.end() &&
+		  par_language->babel() != outer_language->babel()));
+
+	if (closing_rtl_ltr_environment || (boost::next(pit) == paragraphs.end()
+	    && par_language->babel() != outer_language->babel())) {
 		// Since \selectlanguage write the language to the aux file,
 		// we need to reset the language at the end of footnote or
 		// float.
@@ -468,11 +536,11 @@ TeXOnePar(Buffer const & buf,
 			texrow.newline();
 		}
 		if (lyxrc.language_command_end.empty()) {
-			if (!doc_language->babel().empty()) {
+			if (!prev_language->babel().empty()) {
 				os << from_ascii(subst(
 					lyxrc.language_command_begin,
 					"$$lang",
-					doc_language->babel()));
+					prev_language->babel()));
 				pending_newline = true;
 			}
 		} else if (!par_language->babel().empty()) {
@@ -483,12 +551,31 @@ TeXOnePar(Buffer const & buf,
 			pending_newline = true;
 		}
 	}
+	if (closing_rtl_ltr_environment)
+		os << "}";
 
 	if (pending_newline) {
 		os << '\n';
 		texrow.newline();
 	}
-	runparams_in.encoding = runparams.encoding;
+
+	// If this is the last paragraph, and a local_font was set upon entering
+	// the inset, the encoding should be set back to that local_font's 
+	// encoding. We don't use switchEncoding(), because no explicit encoding
+	// switch command is needed, since latex will automatically revert to it
+	// when this inset closes.
+	// This switch is only necessary if we're using "auto" or "default" 
+	// encoding. 
+	if (boost::next(pit) == paragraphs.end() && runparams_in.local_font != 0) {
+		runparams_in.encoding = runparams_in.local_font->language()->encoding();
+		if (bparams.inputenc == "auto" || bparams.inputenc == "default")
+			os << setEncoding(runparams_in.encoding->iconvName());
+
+	}
+	// Otherwise, the current encoding should be set for the next paragraph.
+	else
+		runparams_in.encoding = runparams.encoding;
+
 
 	// we don't need it for the last paragraph!!!
 	// Note from JMarc: we will re-add a \n explicitely in