ctest export tests: document requirements.

Add suggestion for policy for "reuse" of documentation files.

development/autotests/unreliableTests

@@ -6,43 +6,43 @@
 # Matching tests get the label "unreliable" and do not get the label "export".
 #
 # TODO: Name problem: *invalid* tests (wrong output) are not *unreliable*.
-#       Use "unfit" as better label and name of pattern file?
+#   Use "unfit" or "unapplicable" as better label and name of pattern file?
 
 Sublabel: nonstandard
 #
 # Documents with additional requirements,
-# e.g. a class or package file not on CTAN.
+# e.g. a class or package file not in TeXLive.
 # TODO: rename to "extra"?
 
 export/templates/IUCr-article_(dvi|pdf).*
-export/templates/ja_beamer-conference-ornate-20min_(pdf2|pdf[45]_(texF|systemF))
+# does not work with pdflatex, lualatex (PDF) or XeTeX (postscript specials?).
-export/templates/es_beamer-conference-ornate-20min_(pdf2|pdf[45]_(texF|systemF))
+export/templates/(ja|es)_beamer-conference-ornate-20min_(pdf2|pdf[45]).*
 export/templates/kluwer_.*_systemF
-export/examples/modernCV_pdf4_(tex|system)F
+export/templates/ectaart_(dvi3|pdf5).*
-export/templates/ectaart_(dvi3|pdf5)_(tex|system)F
 
 # 1.) missing farsi package with lfeenc.def
 # 2.) LuaTeX does not support Farsi yet. See:
 #     https://github.com/reutenauer/polyglossia/commit/ccb0e9e2c6411170ad779b05ff5076f1193cc323
 export/examples/fa/splash_(dvi|pdf|pdf[23]|(dvi3|pdf4|pdf5)_(texF|systemF))
 
-# requires knitr, requires rjournal.sty (not on CTAN)
+# requires knitr, requires rjournal.sty (not in TeXLive)
 # incompatible with fontspec?
 export/templates/RJournal_(dvi3|pdf[45])_systemF
 
-# Requires aa.cls (not on CTAN).
+# Requires aa.cls (not in TeXLive).
 # Fails with XeTeX/LuaTeX
 # Reported upstream (https://www.mail-archive.com/lyx-devel@lists.lyx.org/msg184435.html)
 # Is there a new version fixing the problem?
 export/examples/aa_sample_(dvi3|pdf[45])_.*
 export/templates/aa_(dvi3|pdf[45])_.*
 
-# Requires iucr.cls (not on CTAN)
+# Requires iucr.cls (not in TeXLive)
 # Is there a new version fixing the problem?
 # Does LuaTeX produce sensible results?
 export/templates/IUCr-article_pdf4_systemF
 
-# Requires acmsiggraph (not on CTAN)
+# Requires acmsiggraph.cls (not in TeXLive)
+# see http://wiki.lyx.org/Examples/AcmSiggraph
 export/templates/ACM-siggraph_(dvi|pdf).*
 
 Sublabel: erratic

lib/doc/Development.lyx

@@ -1,5 +1,5 @@
 #LyX 2.2 created this file. For more info see http://www.lyx.org/
-\lyxformat 506
+\lyxformat 504
 \begin_document
 \begin_header
 \save_transient_properties true
@@ -1547,8 +1547,8 @@ status collapsed
 \end_inset
 
  command line option.
- The test checks for errors reported by LyX (However, error-free export
+ The tests checks for errors reported by LyX.
- is no guarantee for an error-free output document.)
+ (However, error-free export is no guarantee for an error-free output document.)
 \end_layout
 
 \begin_layout Paragraph
@@ -1569,6 +1569,67 @@ When making a non-trivial change to a .layout file, run the export and layout
 \begin_layout Itemize
 When making non-trivial changes to a .lyx file, run the export tests correspondin
 g to that .lyx file.
+ 
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+This rule is due to revision.
+ 
+\end_layout
+
+\begin_layout Plain Layout
+There is an objection from the documentation maintainer that working on
+ the documentation must not be complicated by having to consider non-standard
+ exports.
+\end_layout
+
+\begin_layout Itemize
+successful compiling/testing an edited documentation file with pdflatex
+ suffices to ensure it can be commited, not tests with other exports are
+ required.
+\end_layout
+
+\begin_layout Plain Layout
+If sudden failures with other exports due to “half-tested” documentation
+ updates are a problem for the test maintainer, the test suite should use
+ copies that are 
+\end_layout
+
+\begin_layout Itemize
+copied to a cache dir (autotests/samples/doc/, say) but not changed,
+\end_layout
+
+\begin_layout Itemize
+updated regularely (but on a time chosen by the test suite maintainer) from
+ the originals in lib/doc/
+\end_layout
+
+\begin_layout Plain Layout
+This way, 
+\end_layout
+
+\begin_layout Itemize
+no test will fail due to ongoing work on documentation,
+\end_layout
+
+\begin_layout Itemize
+the documentation is still tested in full (with some delay),
+\end_layout
+
+\begin_layout Itemize
+failures with non-default export can be examined and handled accordingly
+ in one run with the cache update,
+\end_layout
+
+\begin_layout Itemize
+“interesting failures” (like the nested-language+polyglossia problem in
+ es/Customization can be separated and moved into dedicated test samples.
+\end_layout
+
+\end_inset
+
+
 \end_layout
 
 \begin_layout Itemize
@@ -1906,6 +1967,17 @@ ctest
 
 \end_inset
 
+.
+ A full, up-to-date TeXLive installation is recommended to run the tests.
+ Otherwise, some tests will fail.
+ Tests with additional requirements are labeled 
+\begin_inset Quotes eld
+\end_inset
+
+unreliable:nonstandard
+\begin_inset Quotes erd
+\end_inset
+
 .
  
 \end_layout
@@ -2573,6 +2645,20 @@ reference "par:Unreliable-tests"
 ).
 \end_layout
 
+\begin_layout Itemize
+In case of tests failing due to missing requirements (when only a subset
+ of TeXLive is installed or a test labeled 
+\begin_inset Quotes eld
+\end_inset
+
+unreliable:nonstandard
+\begin_inset Quotes erd
+\end_inset
+
+ fails), ignore the failure, ask for someone else to run the test, or install
+ the missing ressources and try again.
+\end_layout
+
 \begin_layout Paragraph
 \begin_inset CommandInset label
 LatexCommand label