Introduce "inherit" encoding for latex_language

This gets rid of the hardcoded latin1 encoding for verbatim. Instead,
verbatim now inherits the encoding from the context, which is what is
actually wanted here.

Fixes: #9012, #9258

lib/encodings

@@ -246,3 +246,9 @@ End
 # Pure 7bit ASCII encoding (partially hardcoded in LyX)
 Encoding ascii ascii "ASCII" ascii fixed none
 End
+
+# Semantic encodings
+
+# Inherit encoding of the context (used by verbatim)
+Encoding inherit inherit "" "" fixed none
+End

lib/languages

@@ -65,6 +65,8 @@
 # * Encoding is the default encoding used with TeX fonts.
 #   It is only used if Document > Settings > Language > Encoding
 #   is set to "Language Default" and "use non-TeX fonts" is FALSE.
+#   Encoding "inherit" means: keep encoding of the context (used by
+#   latex_language).
 # * InternalEncoding is used to tell LyX that babel internally sets a
 #   non-standard font encoding (such as hebrew to LHE or greek to LGR).
 #   If True, LyX cares for characters/macros that do not exist in
@@ -100,13 +102,13 @@ Language ignore
 	GuiName          "Ignore"
 	BabelName        ignore
 	PolyglossiaName  ignore
-	Encoding         iso8859-1
+	Encoding         inherit
 	LangCode         ignore
 End
 
 Language latex
 	GuiName          "LaTeX"
-	Encoding         iso8859-1
+	Encoding         inherit
 	LangCode         latex
 End
 

src/Cursor.cpp

@@ -21,7 +21,6 @@
 #include "Cursor.h"
 #include "CutAndPaste.h"
 #include "DispatchResult.h"
-#include "Encoding.h"
 #include "Font.h"
 #include "FuncCode.h"
 #include "FuncRequest.h"
@@ -2138,22 +2137,6 @@ docstring Cursor::getPossibleLabel() const
 }
 
 
-Encoding const * Cursor::getEncoding() const
-{
-	if (empty())
-		return 0;
-	BufferParams const & bp = bv().buffer().params();
-	if (bp.useNonTeXFonts)
-		return encodings.fromLyXName("utf8-plain");
-
-	CursorSlice const & sl = innerTextSlice();
-	Text const & text = *sl.text();
-	Font font = text.getPar(sl.pit()).getFont(bp, sl.pos(),
-	                                          text.outerFont(sl.pit()));
-	return font.language()->encoding();
-}
-
-
 void Cursor::undispatched() const
 {
 	disp_.dispatched(false);

src/Cursor.h

@@ -72,7 +72,6 @@ class Row;
 
 // these should go
 class InsetMathUnknown;
-class Encoding;
 
 /**
  * This class describes the position of a cursor within a document,
@@ -546,8 +545,6 @@ public:
 	int macroNamePos();
 	/// can we enter the inset?
 	bool openable(MathAtom const &) const;
-	///
-	Encoding const * getEncoding() const;
 	/// font at cursor position
 	Font getFont() const;
 };

src/DocIterator.cpp

@@ -15,7 +15,11 @@
 #include "DocIterator.h"
 
 #include "Buffer.h"
+#include "BufferParams.h"
+#include "Encoding.h"
+#include "Font.h"
 #include "InsetList.h"
+#include "Language.h"
 #include "Paragraph.h"
 #include "LyXRC.h"
 #include "Text.h"
@@ -688,6 +692,33 @@ void DocIterator::append(DocIterator::idx_type idx, pos_type pos)
 }
 
 
+Encoding const * DocIterator::getEncoding() const
+{
+	if (empty())
+		return 0;
+	BufferParams const & bp = buffer()->params();
+	if (bp.useNonTeXFonts)
+		return encodings.fromLyXName("utf8-plain");
+
+	CursorSlice const & sl = innerTextSlice();
+	Text const & text = *sl.text();
+	Font font = text.getPar(sl.pit()).getFont(bp, sl.pos(),
+	                                          text.outerFont(sl.pit()));
+	Encoding const * enc = font.language()->encoding();
+	if (enc->name() == "inherit") {
+		size_t const n = depth();
+		for (size_t i = 0; i < n; ++i) {
+			Text const & otext = *slices_[i].text();
+			Font ofont = otext.getPar(slices_[i].pit()).getFont(bp, slices_[i].pos(),
+								  otext.outerFont(slices_[i].pit()));
+			if (ofont.language()->encoding()->name() != "inherit")
+				return ofont.language()->encoding();
+		}
+	}
+	return font.language()->encoding();
+}
+
+
 ostream & operator<<(ostream & os, DocIterator const & dit)
 {
 	for (size_t i = 0, n = dit.depth(); i != n; ++i)

src/DocIterator.h

@@ -20,6 +20,7 @@
 namespace lyx {
 
 class DocIterator;
+class Encoding;
 class LyXErr;
 class MathAtom;
 class Paragraph;
@@ -257,6 +258,9 @@ public:
 	/// push one CursorSlice on top and set its index and position
 	void append(idx_type idx, pos_type pos);
 
+	///
+	Encoding const * getEncoding() const;
+
 private:
 	friend class InsetIterator;
 	friend DocIterator doc_iterator_begin(Buffer const * buf, Inset const * inset);

src/Text.cpp

@@ -933,15 +933,12 @@ bool canInsertChar(Cursor const & cur, char_type c)
 		}
 	}
 
-	// Prevent to insert uncodable characters in verbatim and ERT
-	// (workaround for bug 9012)
-	// Don't do it for listings inset, since InsetListings::latex() tries
-	// to switch to a usable encoding which works in many cases (bug 9102).
-	if (par.isPassThru() && cur.inset().lyxCode() != LISTINGS_CODE &&
-	    cur.current_font.language()) {
-		Encoding const * e = cur.current_font.language()->encoding();
+	// Prevent to insert uncodable characters in verbatim and ERT.
+	// The encoding is inherited from the context here.
+	if (par.isPassThru() && cur.getEncoding()) {
+		Encoding const * e = cur.getEncoding();
 		if (!e->encodable(c)) {
-			cur.message(_("Character is uncodable in verbatim paragraphs."));
+			cur.message(_("Character is uncodable in this verbatim context."));
 			return false;
 		}
 	}

src/frontends/qt4/GuiDocument.cpp

@@ -1076,11 +1076,10 @@ GuiDocument::GuiDocument(GuiView & lv)
 	// Always put the default encoding in the first position.
 	langModule->encodingCO->addItem(qt_("Language Default (no inputenc)"));
 	QStringList encodinglist;
-	Encodings::const_iterator it = encodings.begin();
-	Encodings::const_iterator const end = encodings.end();
-	for (; it != end; ++it)
-		if (!it->unsafe())
-			encodinglist.append(qt_(it->guiName()));
+	for (auto const & encvar : encodings) {
+		if (!encvar.unsafe() && !encvar.guiName().empty())
+			encodinglist.append(qt_(encvar.guiName()));
+	}
 	encodinglist.sort();
 	langModule->encodingCO->addItems(encodinglist);
 

src/output_latex.cpp

@@ -1488,7 +1488,7 @@ pair<bool, int> switchEncoding(odocstream & os, BufferParams const & bparams,
 	// * with useNonTeXFonts: "utf8plain",
 	// * with XeTeX and TeX fonts: "ascii" (inputenc fails),
 	// * with LuaTeX and TeX fonts: only one encoding accepted by luainputenc.
-	if (runparams.isFullUnicode())
+	if (runparams.isFullUnicode() || newEnc.name() == "inherit")
 		return make_pair(false, 0);
 
 	Encoding const & oldEnc = *runparams.encoding;