Cmake export tests: Renamed nonstandardTests to unreliableTests

and introduce sublabels

Sublabels section in *.Tests starts with 'Sublabel: name'
and is valid until start of a new sublabel.
'name' contains only ascii characters [a-z]+
This commit is contained in:
Kornel Benko 2015-11-24 18:42:36 +01:00
parent 1f0305509b
commit 05b517e614
2 changed files with 87 additions and 30 deletions

View File

@ -63,41 +63,60 @@ macro(getoutputformats filepath varname)
set(${varname} ${out_formats})
endmacro()
macro(findexpr found testname listname)
macro(findexpr found testname listname rsublabel)
set(_found 0)
set(sublabel "")
foreach(_itrx ${${listname}})
set(_itr "^${_itrx}$")
if(${testname} MATCHES ${_itr})
set(_found 1)
break()
if ("${_itrx}" MATCHES "^Sublabel:[ \t]*\([a-z]+\)[ \t]*$")
set(sublabel "${CMAKE_MATCH_1}")
else()
set(_itr "^${_itrx}$")
if (${testname} MATCHES "${_itr}")
set(_found 1)
break()
endif()
endif()
endforeach()
if (${_found})
if (NOT "${sublabel}" STREQUAL "")
set(${rsublabel} ${sublabel})
endif()
endif()
set(${found} ${_found})
endmacro()
macro(maketestname testname reverted listreverted listignored listnonstandard)
macro(maketestname testname reverted listreverted listignored listunreliable listlabels)
set(sublabel "")
string(REGEX MATCH "\\/[a-z][a-z](_[A-Z][A-Z])?\\/" _v ${${testname}})
if(_v)
string(REGEX REPLACE "\\/" "" _v ${_v})
set(listrevertedx ${listreverted}_${_v})
set(listignoredx ${listignored}_${_v})
set(listnonstandardx ${listnonstandard}_${_v})
set(listunreliablex ${listunreliable}_${_v})
else()
set(listrevertedx ${listreverted})
set(listignoredx ${listignored})
set(listnonstandardx ${listnonstandard})
set(listunreliablex ${listunreliable})
endif()
findexpr(mfound ${testname} ${listignoredx})
findexpr(mfound ${testname} ${listignoredx} sublabel)
if (NOT mfound)
# check if they are non-standard
findexpr(nsfound ${testname} ${listnonstandardx})
# check if they are unreliable
set(sublabel "")
findexpr(nsfound ${testname} ${listunreliablex} sublabel)
if (nsfound)
set(${testname} "NON_STANDARD.${${testname}}")
if (NOT "${sublabel}" STREQUAL "")
list(APPEND ${listlabels} ${sublabel})
string(TOUPPER "${sublabel}_" tmpprefix)
else()
set(tmpprefix "")
endif()
set(${testname} "UNRELIABLE.${tmpprefix}${${testname}}")
else ()
string(REGEX MATCH "_(systemF|texF|pdf3|pdf2|pdf|dvi|lyx16|xhtml)$" _v ${${testname}})
set(sublabel "")
# check if test _may_ be in listreverted
if (_v)
findexpr(mfound ${testname} ${listrevertedx})
findexpr(mfound ${testname} ${listrevertedx} sublabel)
else()
set(mfound OFF)
endif()
@ -106,6 +125,9 @@ macro(maketestname testname reverted listreverted listignored listnonstandard)
else()
set(${reverted} 1)
set(${testname} "INVERTED_SEE-README.ctest_${${testname}}")
if (NOT sublabel STREQUAL "")
list(APPEND ${listlabels} ${sublabel})
endif()
endif()
endif()
else()
@ -115,18 +137,26 @@ endmacro()
macro(loadTestList filename resList)
# Create list of strings from a file without comments
file(STRINGS ${filename} tempList)
file(STRINGS ${filename} tempList ENCODING "UTF-8")
set(${resList})
set(sublabel)
foreach(_l ${tempList})
string(REGEX REPLACE "[ \t]*#.*" "" _newl "${_l}")
set(_newl "${_l}")
string(REGEX REPLACE "[ \t]*#.*$" "" _newl "${_l}")
if(_newl)
#message(STATUS "Testing ${_newl}")
string(REGEX REPLACE "(\\/|\\||\\(|\\))" " " _vxx ${_newl})
string(REGEX MATCHALL " ([a-z][a-z](_[A-Z][A-Z])?) " _vx ${_vxx})
if (_newl MATCHES "^Sublabel:")
set(sublabel ${_newl})
endif()
if(_vx)
foreach(_v ${_vx})
string(REGEX REPLACE " " "" _v ${_v})
#message(STATUS " ==> ${resList}_${_v}")
if (NOT ${sublabel} STREQUAL "${sublabel}_${_v}")
list(APPEND ${resList}_${_v} "${sublabel}")
set(${sublabel}_${_v} "${sublabel}")
endif()
list(APPEND ${resList}_${_v} "${_newl}")
endforeach()
if(_newl MATCHES "\\(\\|")
@ -144,21 +174,26 @@ endmacro()
loadTestList(revertedTests revertedTests)
loadTestList(ignoredTests ignoredTests)
loadTestList(suspendedTests suspendedTests)
loadTestList(nonstandardTests nonstandardTests)
loadTestList(unreliableTests unreliableTests)
macro(handlesuspended TestName reverted testlabel)
set(mylabel ${${testlabel}})
set(myreverted ${reverted})
set(sublabel)
# check for nonstandard
if ("${TestName}" MATCHES "^NON_STANDARD")
set(mylabel "nonstandard")
# check for unreliable
if ("${TestName}" MATCHES "^UNRELIABLE")
list(REMOVE_ITEM mylabel "export" "reverted" "templates" "mathmacros" "manuals")
list(APPEND mylabel "unreliable")
set(myreverted 0)
elseif (${reverted})
# check suspension only for reverted tests
findexpr(tfound TestName suspendedTests)
findexpr(tfound TestName suspendedTests sublabel)
if (tfound)
set(mylabel "suspended")
if (sublabel)
list(APPEND mylabel ${sublabel})
endif()
set(myreverted 0) # if test is to be suspended, remove the 'reverted' flag
endif()
endif()
@ -200,7 +235,8 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
# Strip extension
string(REGEX REPLACE "\\.lyx$" "" f ${f})
set(TestName "export/${libsubfolder}/${f}_lyx16")
maketestname(TestName reverted revertedTests ignoredTests nonstandardTests)
set(mytestlabel ${testlabel})
maketestname(TestName reverted revertedTests ignoredTests unreliableTests mytestlabel)
if(TestName)
add_test(NAME ${TestName}
WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
@ -216,13 +252,14 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
-DTOP_SRC_DIR=${TOP_SRC_DIR}
-DPERL_EXECUTABLE=${PERL_EXECUTABLE}
-P "${TOP_SRC_DIR}/development/autotests/export.cmake")
handlesuspended(${TestName} ${reverted} testlabel) # checking for suspended lyx16 exports
handlesuspended(${TestName} ${reverted} mytestlabel) # checking for suspended lyx16 exports
endif()
if(LYX_PYTHON_EXECUTABLE)
set(lyx2lyxtestlabel "lyx2lyx")
# For use of lyx2lyx we need the python executable
set(mytestlabel ${lyx2lyxtestlabel})
set(TestName "lyx2lyx/${libsubfolder}/${f}")
maketestname(TestName reverted revertedTests ignoredTests nonstandardTests)
maketestname(TestName reverted revertedTests ignoredTests unreliableTests mytestlabel)
if(TestName)
add_test(NAME ${TestName}
WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
@ -232,13 +269,14 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
"-DLYX_TESTS_USERDIR=${LYX_TESTS_USERDIR}"
"-DLYXFILE=${LIBSUB_SRC_DIR}/${f}.lyx"
-P "${TOP_SRC_DIR}/development/autotests/lyx2lyxtest.cmake")
handlesuspended(${TestName} ${reverted} lyx2lyxtestlabel)
handlesuspended(${TestName} ${reverted} mytestlabel)
endif()
endif()
set(loadtestlabel "load")
set(mytestlabel ${loadtestlabel})
set(TestName "check_load/${libsubfolder}/${f}")
maketestname(TestName reverted revertedTests ignoredTests nonstandardTests)
maketestname(TestName reverted revertedTests ignoredTests unreliableTests mytestlabel)
if(TestName)
set(loadtestlabel "load")
add_test(NAME ${TestName}
WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
COMMAND ${CMAKE_COMMAND} -DLYXFILE=${LIBSUB_SRC_DIR}/${f}.lyx
@ -247,7 +285,7 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
-DPARAMS_DIR=${TOP_SRC_DIR}/development/autotests
-DWORKDIR=${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}
-P "${TOP_SRC_DIR}/development/autotests/check_load.cmake")
handlesuspended(${TestName} ${reverted} loadtestlabel)
handlesuspended(${TestName} ${reverted} mytestlabel)
set_tests_properties(${TestName} PROPERTIES RUN_SERIAL ON)
endif()
getoutputformats("${LIBSUB_SRC_DIR}/${f}.lyx" formatlist)
@ -263,7 +301,8 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
else()
set(TestName "export/${libsubfolder}/${f}_${format}_${fonttype}")
endif()
maketestname(TestName reverted revertedTests ignoredTests nonstandardTests)
set(mytestlabel ${testlabel})
maketestname(TestName reverted revertedTests ignoredTests unreliableTests mytestlabel)
if(TestName)
add_test(NAME ${TestName}
WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
@ -279,7 +318,7 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
-DTOP_SRC_DIR=${TOP_SRC_DIR}
-DPERL_EXECUTABLE=${PERL_EXECUTABLE}
-P "${TOP_SRC_DIR}/development/autotests/export.cmake")
handlesuspended(${TestName} ${reverted} testlabel) # check for suspended pdf/dvi exports
handlesuspended(${TestName} ${reverted} mytestlabel) # check for suspended pdf/dvi exports
endif()
endforeach()
endforeach()

View File

@ -0,0 +1,18 @@
# Tests using some non-standard tex class
Sublabel: nonstandard
export/templates/IUCr-article_(dvi|pdf).*
export/templates/es_beamer-conference-ornate-20min_pdf4_texF
export/templates/kluwer_pdf[45]_systemF
export/examples/modernCV_pdf4_(tex|system)F
export/templates/ectaart_(dvi3|pdf5)_(tex|system)F
export/examples/fa/splash_(dvi|pdf|pdf[23]|(dvi3|pdf4|pdf5)_(texF|systemF))
#
#
# 1.) missing farsi package with lfeenc.def
# 2.) LuaTeX does not support Farsi yet. See:
# https://github.com/reutenauer/polyglossia/commit/ccb0e9e2c6411170ad779b05ff5076f1193cc323
# Tests depending on local configuration, OS, TeX distribution,
# package versions, or the phase of the moon.
Sublabel: erratic
export/examples/(|fr/)seminar_(dvi|pdf|pdf[23]|pdf4_texF|pdf5_systemF)
#