Properly communicate forced encodings

This is currently only relevant fo InsetListings, which falls back to
a fixed-width encoding under specific conditions. It is now possible
to query the inset about that and report the correct encoding in
DocIterator::getEncoding.

Addresses the second part of #10995

(cherry picked from commit f924ef2966)

src/DocIterator.cpp

@@ -714,6 +714,24 @@ Encoding const * DocIterator::getEncoding() const
 		(customenc && lang->encoding()->package() != Encoding::CJK)
 		? &bp.encoding() : lang->encoding();
 
+	// Some insets force specific encodings sometimes (e.g., listings in
+	// multibyte context forces singlebyte).
+	if (inset().forcedEncoding(enc, encodings.fromLyXName("iso8859-1"))) {
+		// Get the language outside the inset
+		size_t const n = depth();
+		for (size_t i = 0; i < n; ++i) {
+			Text const & otext = *slices_[i].text();
+			Language const * olang =
+					otext.getPar(slices_[i].pit()).getFont(bp, slices_[i].pos(),
+														   otext.outerFont(slices_[i].pit())).language();
+			Encoding const * oenc = olang->encoding();
+			if (oenc->name() != "inherit")
+				return inset().forcedEncoding(enc, oenc);
+		}
+		// Fall back to buffer encoding if no outer lang was found.
+		return inset().forcedEncoding(enc, &bp.encoding());
+	}
+
 	// Inherited encoding (latex_language) is determined by the context
 	// Look for the first outer encoding that is not itself "inherit"
 	if (lang->encoding()->name() == "inherit") {

src/insets/Inset.h

@@ -39,6 +39,7 @@ class Cursor;
 class CursorSlice;
 class Dimension;
 class DocIterator;
+class Encoding;
 class FuncRequest;
 class FuncStatus;
 class InsetArgument;
@@ -417,6 +418,10 @@ public:
 	/// if this inset has paragraphs should they be forced to use a
 	/// local font language switch?
 	virtual bool forceLocalFontSwitch() const { return false; }
+	/// Does the inset force a specific encoding?
+	virtual Encoding const * forcedEncoding(Encoding const *, Encoding const *) const
+	{ return 0; }
+
 
 	/// Is the content of this inset part of the output document?
 	virtual bool producesOutput() const { return true; }

src/insets/InsetListings.cpp

@@ -119,6 +119,26 @@ void InsetListings::read(Lexer & lex)
 }
 
 
+Encoding const * InsetListings::forcedEncoding(Encoding const * inner_enc,
+											   Encoding const * outer_enc) const
+{
+	// The listings package cannot deal with multi-byte-encoded
+	// glyphs, except if full-unicode aware backends
+	// such as XeTeX or LuaTeX are used, and with pLaTeX.
+	// Minted can deal with all encodings.
+	if (buffer().params().use_minted
+		|| (buffer().params().encoding().package() == Encoding::japanese
+			&& inner_enc->package() == Encoding::japanese)
+		|| inner_enc->hasFixedWidth())
+		return 0;
+
+	// We try if there's a singlebyte encoding for the outer
+	// language; if not, fall back to latin1.
+	return (outer_enc->hasFixedWidth()) ?
+			outer_enc : encodings.fromLyXName("iso8859-1");
+}
+
+
 void InsetListings::latex(otexstream & os, OutputParams const & runparams) const
 {
 	string param_string = params().params();
@@ -160,30 +180,19 @@ void InsetListings::latex(otexstream & os, OutputParams const & runparams) const
 
 	bool encoding_switched = false;
 	Encoding const * const save_enc = runparams.encoding;
-	// The listings package cannot deal with multi-byte-encoded
-	// glyphs, except if full-unicode aware backends
-	// such as XeTeX or LuaTeX are used, and with pLaTeX.
-	bool const multibyte_possible =	use_minted || runparams.isFullUnicode()
-	    || (buffer().params().encoding().package() == Encoding::japanese
-	        && runparams.encoding->package() == Encoding::japanese);
 
-	if (!multibyte_possible && !runparams.encoding->hasFixedWidth()) {
+	Encoding const * const outer_encoding =
+		(runparams.local_font != 0) ?
+			runparams.local_font->language()->encoding()
+			: buffer().params().language->encoding();
+	Encoding const * fixedlstenc = forcedEncoding(runparams.encoding, outer_encoding);
+	if (fixedlstenc) {
 		// We need to switch to a singlebyte encoding, due to
 		// the restrictions of the listings package (see above).
 		// This needs to be consistent with
 		// LaTeXFeatures::getTClassI18nPreamble().
-		Language const * const outer_language =
+		switchEncoding(os.os(), buffer().params(), runparams, *fixedlstenc, true);
-			(runparams.local_font != 0) ?
+		runparams.encoding = fixedlstenc;
-				runparams.local_font->language()
-				: buffer().params().language;
-		// We try if there's a singlebyte encoding for the current
-		// language; if not, fall back to latin1.
-		Encoding const * const lstenc =
-			(outer_language->encoding()->hasFixedWidth()) ?
-				outer_language->encoding()
-				: encodings.fromLyXName("iso8859-1");
-		switchEncoding(os.os(), buffer().params(), runparams, *lstenc, true);
-		runparams.encoding = lstenc;
 		encoding_switched = true;
 	}
 
@@ -334,7 +343,7 @@ void InsetListings::latex(otexstream & os, OutputParams const & runparams) const
 	if (!uncodable.empty() && !runparams.silent) {
 		// issue a warning about omitted characters
 		// FIXME: should be passed to the error dialog
-		if (!multibyte_possible && !runparams.encoding->hasFixedWidth())
+		if (fixedlstenc)
 			frontend::Alert::warning(_("Uncodable characters in listings inset"),
 				bformat(_("The following characters in one of the program listings are\n"
 					  "not representable in the current encoding and have been omitted:\n%1$s.\n"

src/insets/InsetListings.h

@@ -79,6 +79,8 @@ private:
 	TexString getCaption(OutputParams const &) const;
 	///
 	bool insetAllowed(InsetCode c) const { return c == CAPTION_CODE || c == QUOTE_CODE; }
+	///
+	Encoding const * forcedEncoding(Encoding const *, Encoding const *) const;
 
 	///
 	InsetListingsParams params_;