Update tests and documentation for supported languages.

autotests/export/latex/language-in-tables.lyx

@@ -0,0 +1,383 @@
+#LyX 2.4 created this file. For more info see https://www.lyx.org/
+\lyxformat 573
+\begin_document
+\begin_header
+\save_transient_properties true
+\origin unavailable
+\textclass article
+\use_default_options true
+\maintain_unincluded_children false
+\language greek
+\language_package default
+\inputencoding auto-legacy
+\fontencoding auto
+\font_roman "default" "default"
+\font_sans "default" "default"
+\font_typewriter "default" "default"
+\font_math "auto" "auto"
+\font_default_family default
+\use_non_tex_fonts false
+\font_sc false
+\font_osf false
+\font_sf_scale 100 100
+\font_tt_scale 100 100
+\use_microtype false
+\use_dash_ligatures true
+\graphics default
+\default_output_format pdf2
+\output_sync 0
+\bibtex_command default
+\index_command default
+\float_placement class
+\float_alignment class
+\paperfontsize default
+\spacing single
+\use_hyperref false
+\papersize default
+\use_geometry false
+\use_package amsmath 1
+\use_package amssymb 1
+\use_package cancel 1
+\use_package esint 1
+\use_package mathdots 1
+\use_package mathtools 1
+\use_package mhchem 1
+\use_package stackrel 1
+\use_package stmaryrd 1
+\use_package undertilde 1
+\cite_engine basic
+\cite_engine_type default
+\biblio_style plain
+\use_bibtopic false
+\use_indices false
+\paperorientation portrait
+\suppress_date false
+\justification true
+\use_refstyle 1
+\use_minted 0
+\index Index
+\shortcut idx
+\color #008000
+\end_index
+\secnumdepth 3
+\tocdepth 3
+\paragraph_separation indent
+\paragraph_indentation default
+\is_math_indent 0
+\math_numbering_side default
+\quotes_style french
+\dynamic_quotes 0
+\papercolumns 1
+\papersides 1
+\paperpagestyle default
+\tablestyle default
+\tracking_changes false
+\output_changes false
+\html_math_output 0
+\html_css_as_file 0
+\html_be_strict false
+\end_header
+
+\begin_body
+
+\begin_layout Subsection*
+
+\lang english
+languages in tables and insets
+\end_layout
+
+\begin_layout Standard
+
+\lang english
+This Greek document with English text parts demonstrates some of the problems
+ with language inside tables and insets.
+\end_layout
+
+\begin_layout Standard
+
+\lang english
+Text not marked up as English will be written in Greek letters
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+
+\lang english
+unless non-TeX fonts are used
+\end_layout
+
+\end_inset
+
+ using the 
+\begin_inset Quotes fld
+\end_inset
+
+LGR transliteration
+\begin_inset Quotes frd
+\end_inset
+
+ (due to Greek letters at the place of Latin ones in the used TeX font),
+ eg.
+ 
+\lang greek
+
+\begin_inset Quotes fld
+\end_inset
+
+logos
+\lang english
+
+\begin_inset Quotes frd
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Standard
+
+\lang english
+However, this also happens for text in a table inside English parts 
+\begin_inset Tabular
+<lyxtabular version="3" rows="1" columns="2">
+<features tabularvalignment="middle">
+<column alignment="center" valignment="top">
+<column alignment="center" valignment="top">
+<row>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+left
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+right column
+\end_layout
+
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+
+\lang english
+unless the Language of the content is explicitely set to English 
+\begin_inset Tabular
+<lyxtabular version="3" rows="1" columns="2">
+<features tabularvalignment="middle">
+<column alignment="center" valignment="top">
+<column alignment="center" valignment="top">
+<row>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+
+\lang english
+left
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+
+\lang english
+right column
+\end_layout
+
+\end_inset
+</cell>
+</row>
+</lyxtabular>
+
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Standard
+
+\lang english
+The same problem is seen with footnote
+\begin_inset Foot
+status open
+
+\begin_layout Plain Layout
+English text not marked up.
+\end_layout
+
+\end_inset
+
+ insets
+\begin_inset Foot
+status open
+
+\begin_layout Plain Layout
+
+\lang english
+marked up English text
+\end_layout
+
+\end_inset
+
+ and marginal notes
+\begin_inset Marginal
+status open
+
+\begin_layout Plain Layout
+English text not marked up
+\end_layout
+
+\end_inset
+
+ as well as 
+\begin_inset Note Greyedout
+status open
+
+\begin_layout Plain Layout
+greyed out text
+\end_layout
+
+\end_inset
+
+.
+\end_layout
+
+\begin_layout Standard
+
+\lang english
+Date inset and filename inset use current language.
+ English:
+\begin_inset Info
+type  "date"
+arg   "loclong"
+\end_inset
+
+, Greek:
+\lang greek
+ 
+\begin_inset Info
+type  "date"
+arg   "loclong"
+\end_inset
+
+
+\lang english
+.
+ The source file is 
+\begin_inset Quotes fld
+\end_inset
+
+
+\begin_inset Info
+type  "buffer"
+arg   "name"
+\end_inset
+
+
+\begin_inset Quotes frd
+\end_inset
+
+
+\lang greek
+
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+
+\lang english
+In Greek text it would be output as 
+\lang greek
+
+\begin_inset Quotes fld
+\end_inset
+
+
+\begin_inset Info
+type  "buffer"
+arg   "name"
+\end_inset
+
+
+\begin_inset Quotes frd
+\end_inset
+
+
+\lang english
+, like any Latin words and abbreviations it must be set to a different language
+ to appear right.
+\end_layout
+
+\end_inset
+
+
+\lang english
+ .
+ So does a hyperlink to the 
+\begin_inset CommandInset href
+LatexCommand href
+name "LyX home"
+target "http://www.lyx.org"
+literal "false"
+
+\end_inset
+
+ site.
+\end_layout
+
+\begin_layout Standard
+
+\lang english
+\begin_inset Float table
+placement b
+alignment document
+wide false
+sideways false
+status open
+
+\begin_layout Plain Layout
+
+\end_layout
+
+\begin_layout Plain Layout
+
+\lang english
+\begin_inset Caption Standard
+
+\begin_layout Plain Layout
+
+\lang english
+Table float with Greek caption prefix in the GUI but English in the output.
+\end_layout
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Plain Layout
+
+\end_layout
+
+\end_inset
+
+
+\end_layout
+
+\end_body
+\end_document

autotests/export/latex/languagenesting_tablecell.lyx

@@ -24,7 +24,7 @@
 \use_microtype false
 \use_dash_ligatures false
 \graphics default
-\default_output_format default
+\default_output_format pdf2
 \output_sync 0
 \bibtex_command default
 \index_command default
@@ -81,8 +81,8 @@
 \begin_body
 
 \begin_layout Standard
-This document did not compile, if there because of the two paragraphs in
+This document did not compile because of language nesting issues with the
- the second column of fixed size (here 5 cm) with German text.
+ two paragraphs in the second table column of fixed size with German text.
 \end_layout
 
 \begin_layout Standard

development/autotests/ignoredTests

@@ -74,7 +74,7 @@ export/examples/Modules/Chessboard_(lyx(16|20|21|22|23)|(pdf3|pdf5|dvi3)_texF)
 export/(doc|examples|templates)/.*_pdf4_texF
 export/(doc|examples|templates)/(|ar|ca|cs|da|de|el|es|eu|fa|fr|gl|he|hu|id|it|ja|ko|nb|nl|pl|pt|pt_BR|pt_PT|ro|sk|sl|sr|sv|uk|zh_CN)/.*pdf4_texF
 # dedicated tests with inputencoding "auto-legacy":
-export/export/latex/languages/supported-languages_pdf4_texF
+export/export/latex/languages/supported-languages_babel_auto-legacy_pdf4_texF
 
 # CJK fails (sometimes) with Xe/LuaTeX and 8-bit fonts:
 export/examples/ja/Multilingual_Typesetting_.*CJK.(dvi3|pdf5)_texF
@@ -174,6 +174,11 @@ lyx2lyx/export/latex/.*
 export/export/latex/.*(lyx..|xhtml)
 
 # don't test with non-TeX fonts:
+export/export/latex/languages/supported-languages_babel_(pdf|dvi)._systemF
+export/export/latex/languages/supported-languages_babel_auto-legacy_.*_systemF
+
+# don't test with TeX fonts:
+export/export/latex/languages/supported-languages_babel_non-tex_.*(dvi|pdf.?|texF)
 
 # HTML
 # don't check loading, lyx2lyx, or export to other LyX versions or DVI/PDF

development/autotests/invertedTests

@@ -74,24 +74,10 @@ export/templates/Articles/International_Union_of_Crystallography.*_pdf4_systemF
 export/templates/Articles/R_Journal_(dvi3|pdf[45])_systemF
 #export/templates/Articles/Kluwer_.*_systemF
 
-# Babel and fontenc fails for some supported languages
+# Babel with non-TeX fonts fails for some supported languages
-# * check BabelPremble definitions in lib/languages
+# Lithuanian: L7x font encoding also set with LuaTeX and non-TeX fonts
-# * fails also due to #11522, see "lyxbugs" below.
+# (error but output OK)
-export/export/latex/languages/supported-languages_.*_systemF
+export/export/latex/languages/supported-languages_babel_non-tex_(dvi3|pdf5)_systemF
-
-# dvipdfm fails with "culmus-latex" fonts, because some of the fonts are
-# type1 *.pfa while dvipdfm only works with *.pfb fonts:
-#   dvipdfmx:warning: Could not locate a virtual/physical font for TFM "david".
-#   dvipdfmx:warning: >> This font is mapped to a physical font "DavidCLM-Medium.pfa".
-#   dvipdfmx:warning: >> Please check if kpathsea library can find this font: DavidCLM-Medium.pfa
-# but: `kpsewhich DavidCLM-Medium.pfa` finds the font!
-#
-# The dvipdfmx log has also many warnings
-# due to the Hebrew vocalisation using Postscript 3 specials
-#  xdvipdfmx:warning: Interpreting PS code failed!!! Output might be broken!!!
-#  xdvipdfmx:warning: Interpreting special command ps: (ps:) failed.
-export/export/latex/languages/supported-languages_pdf3
-
 
 # Thai with CJK package requires the babel option "thaicjk" instead of "thai".
 # (see thaifont.txt in the CJK documentation for setup details).
@@ -131,6 +117,10 @@ export/export/latex/6197-polish-amssymb_pdf._texF
 # The document "sr/Braille.lyx" is an example of case b).
 # (For some reason, the false positive error is not thrown with PDF (luatex).)
 export/examples/sr/Modules/Braille_(dvi3|pdf4)_systemF
+export/export/latex/language-in-tables_.*_systemF
+
+#9633 Nested language handling causes compilation failures with polyglossia
+export/export/latex/languagenesting-quote-branch_.*_systemF
 
 #9871 LyX sends invalid Unicode to iconv when converting to ASCII
 # most probably due to BabelPreamble code (language specific headings for
@@ -180,6 +170,9 @@ export/doc/(|de/|es/|fr/)Math.*systemF
 # minimal example for Math.lyx:
 export/export/latex/systemfonts-Math-missingchars.*systemF
 #
+# Farsi: missing glyphs (em space and "English" quotes)
+export/examples/fa/Welcome_pdf4_systemF
+#
 # CJK-quotes examples (missing in most Latin-script fonts)
 export/doc/(|de/|es/|fr/)UserGuide_(dvi3|pdf4|pdf5)_systemF
 #
@@ -287,6 +280,11 @@ export/templates/Presentations/Slides_.*_systemF
 # Beamer: conflicts with some combinations of packages/settings/languages
 export/templates/es/Presentations/Beamer_pdf4_texF
 
+# Babel-French fails with Xe/LuaTeX and TeX fonts
+#   LaTeX Error: Command \at already defined. (and a lot more)
+export/export/latex/languages/supported-languages_(dvi|pdf).*_texF
+export/export/latex/languages/supported-languages_babel_(dvi|pdf).*_texF
+
 
 # Arabic
 #
@@ -306,34 +304,34 @@ export/templates/es/Presentations/Beamer_pdf4_texF
 # see discussion at https://www.mail-archive.com/search?l=mid&q=20161114035123.s5lt4ib4x4obtptp%40steph
 #
 # Babel-Arabic loads "inputenc" which fails with LuaTeX
-# Wit XeTeX, it works if "utf8" input encoding is set (as in ar/Welcome.lyx)
+# With XeTeX, it works if "utf8" input encoding is set (as in ar/Welcome.lyx)
 !export/examples/ar/Welcome_pdf4_texF
 export/.*/ar/.*_texF
-# Polyglossia bug with LuaTeX and LTR languages (also Hebrew and Farsi):
-# Undefined control sequences "\luatexpardir" and "\luatextextdir".
-export/.*/ar/.*(dvi3|pdf5)_systemF
 # no Arab charaters with luainputenc
 export/export/latex/languages/supported-languages_(dvi3|pdf5)_texF
+export/export/latex/languages/supported-languages_babel_auto-legacy_(dvi3|pdf5)_texF
 
 # Farsi
-# Polyglossia problem with LuaTeX (undefined command) (cf. Arabic)
-# https://github.com/reutenauer/polyglossia/commit/ccb0e9e2c6411170ad779b05ff5076f1193cc323
-export/examples/fa/Welcome_(dvi3|pdf5)_systemF
-# missing glyphs (em space and "English" quotes) with XeTeX + system fonts.
-export/examples/fa/Welcome_pdf4_systemF
 # "invalid utf-8 sequence" with Babel-Farsi, LuaTeX, and TeX fonts.
 export/examples/fa/Welcome_(dvi3|pdf5)_texF
 
 # Hebrew
-# does not currently work with LuaTeX (undefined command) (cf. Arabic)
-# terminal message: "Language hebrew not found in language.dat.lua"
-export/.*/he/.*(dvi3|pdf5)_systemF
-#
 # babel-hebrew loads "rlbabel.def" which fails with LuaTeX + TeX fonts:
 # "! Right-to-Left Support Error: use TeX--XeT or e-TeX engine."
 # (XeTeX works, if inputenc is set to "utf8")
 export/.*/he/.*_texF
 export/export/latex/languages/he-.*(dvi3|pdf5)_texF
+export/export/latex/languages/supported-languages_(dvi3|pdf5)_texF
+export/export/latex/languages/supported-languages_babel_(dvi3|pdf5)_texF
+
+# Polyglossia bug with LuaTeX and LTR languages (Arab, Hebrew and Farsi):
+# Undefined control sequences "\luatexpardir" and "\luatextextdir".
+# https://github.com/reutenauer/polyglossia/commit/ccb0e9e2c6411170ad779b05ff5076f1193cc323
+export/.*/ar/.*(dvi3|pdf5)_systemF
+export/.*/fa/Welcome_(dvi3|pdf5)_systemF
+# terminal message: "Language hebrew not found in language.dat.lua"
+export/.*/he/.*(dvi3|pdf5)_systemF
+export/export/latex/languages/supported-languages_(dvi3|pdf5)_systemF
 
 # Ukrainian
 # See comment in unreliableTests

development/autotests/unreliableTests

@@ -78,17 +78,28 @@ templates/examples/Books/Springer_Monographs.*
 templates/examples/Books/Springer_Contributed_Books.*
 
 # TeXLive does not set up Hebrew 8-bit TeX fonts
-# We use the external package culmus-latex
+# We use the external package "culmus-latex"
 # https://sourceforge.net/projects/ivritex/files/culmus-latex/culmus-latex-0.7/
+# With current "culmus-latex", dvipdfm fails because some of the fonts are
+# type1 *.pfa while dvipdfm only works with *.pfb fonts:
+#   dvipdfmx:warning: Could not locate a virtual/physical font for TFM "david".
+#   dvipdfmx:warning: >> This font is mapped to a physical font "DavidCLM-Medium.pfa".
+#   dvipdfmx:warning: >> Please check if kpathsea library can find this font: DavidCLM-Medium.pfa
+# but: `kpsewhich DavidCLM-Medium.pfa` finds the font!
+# Converting with `t1binary` (from t1utils) helps.
 export/export/latex/unicodesymbols/.*_he.*pdf2
 export/export/latex/(supported-languages|he-scrarticle).*
 export/.*/he/.*((dvi|pdf.?)|_texF)
+export/export/latex/languages/supported-languages_.*_pdf3
+
 
 # Japanese (platex)
-# dvips requires special font setup for kanji, otherwise
+# dvips requires special font setup for kanji.
+# Debian's texlive-lang-japanese does this automatically, tlmgr not.
+# Without proper font setup setup,
 # * the Postscript file has wrong and missing characters
-#   (without error or warning)
+#   (without error or warning),
-# * as a consequence, ps2pdf fails for some documents on TL18 and TL19:
+# * as a consequence, `ps2pdf` fails for some documents:
 export/.*/ja/.*_pdf