fix #10199 Non-encodable characters with XeTeX and utf8-plain

XeTeX with TeX fonts is only safe with ASCII input encoding (see #9740)
and we therefore force "ascii" when exporting with XeTeX and 8-bit TeX-fonts.

However, "utf8-plain" is a "power-user" option, which allows to switch off LyX's
encoding of the LaTeX file:
keep this also for "XeTeX with TeX fonts".

The user is responsible to ensure all characters can be processed and are
correctly shown in the output. The provided test sample shows the problems
with this encoding without special measures (like loading fontspec in the
user-preamble or a document class).

(cherry picked from commit b170b6e40f

I just converted the test to format 508. Thanks Scott for running the
tests. -Guillaume)

autotests/export/utf8-plain-with-tex-fonts.lyx

@@ -0,0 +1,86 @@
+#LyX 2.2 created this file. For more info see http://www.lyx.org/
+\lyxformat 508
+\begin_document
+\begin_header
+\save_transient_properties true
+\origin unavailable
+\textclass article
+\use_default_options true
+\begin_modules
+logicalmkup
+\end_modules
+\maintain_unincluded_children false
+\language english
+\language_package default
+\inputencoding utf8-plain
+\fontencoding global
+\font_roman "default" "default"
+\font_sans "default" "default"
+\font_typewriter "default" "default"
+\font_math "auto" "auto"
+\font_default_family default
+\use_non_tex_fonts false
+\font_sc false
+\font_osf false
+\font_sf_scale 100 100
+\font_tt_scale 100 100
+\graphics default
+\default_output_format default
+\output_sync 0
+\bibtex_command default
+\index_command default
+\paperfontsize default
+\spacing single
+\use_hyperref false
+\papersize default
+\use_geometry false
+\use_package amsmath 1
+\use_package amssymb 1
+\use_package cancel 1
+\use_package esint 1
+\use_package mathdots 1
+\use_package mathtools 1
+\use_package mhchem 1
+\use_package stackrel 1
+\use_package stmaryrd 1
+\use_package undertilde 1
+\cite_engine basic
+\cite_engine_type default
+\biblio_style plain
+\use_bibtopic false
+\use_indices false
+\paperorientation portrait
+\suppress_date false
+\justification true
+\use_refstyle 1
+\index Index
+\shortcut idx
+\color #008000
+\end_index
+\secnumdepth 3
+\tocdepth 3
+\paragraph_separation indent
+\paragraph_indentation default
+\quotes_language english
+\papercolumns 1
+\papersides 1
+\paperpagestyle default
+\tracking_changes false
+\output_changes false
+\html_math_output 0
+\html_css_as_file 0
+\html_be_strict false
+\end_header
+
+\begin_body
+
+\begin_layout Standard
+¹²³¼½¬¸@ł€¶←
+\end_layout
+
+\begin_layout Standard
+ätsch Grüße, em- and en-dash: — ―
+\end_layout
+
+\end_body
+\end_document

development/autotests/suspiciousTests

@@ -120,6 +120,11 @@ export/doc/(de|es|fr)/Additional_pdf4_texF
 # \ding{58} and \ding{171} not supported with XeTeX and system fonts:
 export/examples/fr/ListesPuces_pdf4_systemF
 
+# inputencoding="utf8-plain" with Xe/LuaTeX: characters with
+# Unicode point > 256 lead to errors with 8-bit fonts
+# (We keep this option for power users setting Unicode fonts in the preamble
+# or document class file).
+export/export/utf8-plain-with-tex-fonts_.*_texF
 
 Sublabel: texissues
 #

development/autotests/unreliableTests

@@ -92,3 +92,7 @@ export/doc/es/.*_(pdf[45]|dvi3)_texF
 # Non-ASCII char in verbatim environment is dropped (with a warning when
 # doing this interactively) with ASCII (and hence also with XeTeX).
 export/doc/(|de/|es/|fr/)EmbeddedObjects_pdf4_texF
+
+# inputencoding="utf8-plain" with 8-bit TeX: non-ASCII characters are passed
+#  as two 8-bit characters leading to garbage in the output
+export/export/utf8-plain-with-tex-fonts_(dvi|pdf.?)

src/Buffer.cpp

@@ -1660,9 +1660,11 @@ bool Buffer::makeLaTeXFile(FileName const & fname,
 {
 	OutputParams runparams = runparams_in;
 
-	// XeTeX with TeX fonts is only safe with ASCII encoding,
+	// XeTeX with TeX fonts is only safe with ASCII encoding (see also #9740),
-	// but the "flavor" is not known in BufferParams::encoding().
+	// Check here, because the "flavor" is not known in BufferParams::encoding()
-	if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX))
+	// (power users can override this safety measure selecting "utf8-plain").
+	if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
+	    && (runparams.encoding->name() != "utf8-plain"))
 		runparams.encoding = encodings.fromLyXName("ascii");
 
 	string const encoding = runparams.encoding->iconvName();
@@ -1748,8 +1750,10 @@ void Buffer::writeLaTeXSource(otexstream & os,
 	OutputParams runparams = runparams_in;
 
 	// XeTeX with TeX fonts is only safe with ASCII encoding,
-	// but the "flavor" is not known in BufferParams::encoding().
+	// Check here, because the "flavor" is not known in BufferParams::encoding()
-	if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX))
+	// (power users can override this safety measure selecting "utf8-plain").
+	if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
+	    && (runparams.encoding->name() != "utf8-plain"))
 		runparams.encoding = encodings.fromLyXName("ascii");
 	// FIXME: when only the current paragraph is shown, this is ignored
 	//        (or not reached) and characters encodable in the current

src/BufferParams.cpp

@@ -2967,8 +2967,8 @@ void BufferParams::writeEncodingPreamble(otexstream & os,
 	// XeTeX/LuaTeX: (see also #9740)
 	// With Unicode fonts we use utf8-plain without encoding package.
 	// With TeX fonts, we cannot use utf8-plain, but "inputenc" fails.
-	// XeTeX must use ASCII encoding, for LuaTeX, we load
+	// XeTeX must use ASCII encoding (see Buffer.cpp),
-	// "luainputenc" (see below).
+	//  for LuaTeX, we load "luainputenc" (see below).
 	if (useNonTeXFonts || features.runparams().flavor == OutputParams::XETEX)
 		return;