make test environment. Exports with luatex and xetex.

In successful cooperation with Scott Kostyshak.
We provide many lyx-documents which are not compilable with luatex or xetex.
But some of them compile, if we change the font use to non_tex_fonts.
Since this would change the appropriate source, we have to convert
it first into a save location. To make it there compilable,
we have to convert also all file references.
languages     used font
he|el|ru|uk  'FreeSans'
fa           'FreeFarsi'
zh_CN        'WenQuanYi Micro Hei'
The whole job is done with a perl script.
This commit is contained in:
Kornel Benko 2013-11-17 23:17:27 +01:00
parent a3d685c16c
commit e690cfc47c
5 changed files with 737 additions and 8 deletions

View File

@ -101,9 +101,14 @@ if(Q_WS_X11)
endif()
endif()
find_package(Perl)
macro(getoutputformats filepath varname)
file(STRINGS "${filepath}" lines)
set(out_formats "xhtml" "pdf" "pdf2" "pdf4" "pdf5")
set(out_formats "xhtml" "dvi" "dvi3" "pdf" "pdf2" "pdf3" "pdf4" "pdf5")
if(NOT PERL_FOUND)
list(REMOVE_ITEM out_formats "pdf4" "pdf5")
endif()
set(${varname} ${out_formats})
foreach(_l ${lines})
if(_l MATCHES "^\\\\default_output_format +\([^ ]+\)")
@ -182,6 +187,8 @@ foreach(libsubfolder doc examples templates)
-Dextension=16.lyx
-Dfile=${f}
-Dreverted=${reverted}
-DTOP_SRC_DIR=${TOP_SRC_DIR}
-DPERL_EXECUTABLE=${PERL_EXECUTABLE}
-P "${TOP_SRC_DIR}/development/autotests/export.cmake")
setmarkedtestlabel(${TestName} ${reverted} "export")
endif()
@ -224,6 +231,8 @@ foreach(libsubfolder doc examples templates)
-Dextension=${format}
-Dfile=${f}
-Dreverted=${reverted}
-DTOP_SRC_DIR=${TOP_SRC_DIR}
-DPERL_EXECUTABLE=${PERL_EXECUTABLE}
-P "${TOP_SRC_DIR}/development/autotests/export.cmake")
setmarkedtestlabel(${TestName} ${reverted} "export")
endif()

View File

@ -21,14 +21,34 @@
# -Dextension=xxx \
# -Dfile=xxx \
# -Dreverted=[01] \
# -DTOP_SRC_DIR=${TOP_SRC_DIR}
# -DPERL_EXECUTABLE=${PERL_EXECUTABLE}
# -P "${TOP_SRC_DIR}/development/autotests/export.cmake"
#
message(STATUS "Executing ${lyx} -userdir \"${WORKDIR}/.lyx\" -E ${format} ${file}.${extension} \"${LYX_ROOT}/${file}.lyx\"")
set(Perl_Script "${TOP_SRC_DIR}/development/autotests/useSystemFonts.pl")
if(format MATCHES "pdf4|pdf5")
message(STATUS "LYX_TESTS_USERDIR = ${LYX_TESTS_USERDIR}")
message(STATUS "Converting with perl ${Perl_Script}")
set(LYX_SOURCE "${WORKDIR}/${file}_${format}.lyx")
message(STATUS "Using source \"${LYX_ROOT}/${file}.lyx\"")
message(STATUS "Using dest \"${LYX_SOURCE}\"")
execute_process(COMMAND ${PERL_EXECUTABLE} "${Perl_Script}" "${LYX_ROOT}/${file}.lyx" "${LYX_SOURCE}" ${format}
RESULT_VARIABLE _err)
string(COMPARE EQUAL ${_err} 0 _erg)
if(NOT _erg)
message(FATAL_ERROR "Export failed while converting")
endif()
else()
message(STATUS "Not converting")
set(LYX_SOURCE "${LYX_ROOT}/${file}.lyx")
endif()
message(STATUS "Executing ${lyx} -userdir \"${LYX_TESTS_USERDIR}\" -E ${format} ${file}.${extension} \"${LYX_SOURCE}\"")
set(ENV{${LYX_USERDIR_VER}} "${LYX_TESTS_USERDIR}")
execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${file}.${extension})
execute_process(
COMMAND ${lyx} -userdir "${LYX_TESTS_USERDIR}" -E ${format} ${file}.${extension} "${LYX_ROOT}/${file}.lyx"
COMMAND ${lyx} -userdir "${LYX_TESTS_USERDIR}" -E ${format} ${file}.${extension} "${LYX_SOURCE}"
RESULT_VARIABLE _err)
if(reverted)
string(COMPARE EQUAL ${_err} 0 _erg)

View File

@ -0,0 +1,357 @@
#! /usr/bin/env perl
# -*- mode: perl; -*-
package lyxStatus;
use strict;
our(@EXPORT, @ISA);
BEGIN {
use Exporter ();
@ISA = qw(Exporter);
@EXPORT = qw(initLyxStack checkLyxLine closeLyxStack diestack);
}
my @stack = (); # list of HASH-Arrays
my $rFont = {};
# The elements are:
# type (layout, inset, header, preamble, ...)
# name
# matching list of matching spes
# search: regular expression
# ext: list of extensions needed for the full path of the file spec
# filetype: one of prefix_only,replace_only,copy_only,prefix_for_list,interpret
# fileidx: index into the resulting array, defining the filename
# result: conatenation of the elements should reflect the parsed line
# but first set the modified value into $result->[$fileidx]
# numerical value will be replaced with appropriate matching group value
sub initLyxStack($)
{
$rFont = $_[0];
$stack[0] = { type => "Starting"};
}
sub diestack($)
{
my ($msg) = @_;
# Print stack
print "Called stack\n";
my @call_stack = ();
for my $depth ( 0 .. 100) {
#my ($pkg, $file, $line, $subname, $hasargs, $wantarray) = caller($depth)
my @stack = caller($depth);
last if ($stack[0] ne "main");
push(@call_stack, \@stack);
}
for my $depth ( 0 .. 100) {
last if (! defined($call_stack[$depth]));
my $subname = $call_stack[$depth]->[3];
my $line = $call_stack[$depth]->[2];
print "($depth) $subname()";
if ($depth > 0) {
my $oldline = $call_stack[$depth-1]->[2];
print ":$oldline";
}
print " called from ";
if (defined($call_stack[$depth+1])) {
my $parent = $call_stack[$depth+1]->[3];
print "$parent():$line\n";
}
else {
my $file = $call_stack[$depth]->[1];
print "\"$file\":$line\n";
}
}
die($msg);
}
sub closeLyxStack()
{
&diestack("Stack not OK") if ($stack[0]->{type} ne "Starting");
}
sub setMatching($)
{
my ($match) = @_;
$stack[0]->{"matching"} = $match;
}
sub getMatching()
{
return($stack[0]->{"matching"});
}
###########################################################
#
sub checkForEndBlock($)
{
my ($l) = @_;
for my $et ( qw( layout inset preamble header)) {
if ($l =~ /^\\end_$et$/) {
&diestack("Not in $et") if ($stack[0]->{type} ne "$et");
#print "End $et\n";
shift(@stack);
return(1);
}
}
return(0);
}
sub newMatch($$)
{
my %elem = @_;
if (! defined($elem{"ext"})) {
$elem{"ext"} = "";
}
if (! defined($elem{"filetype"})) {
$elem{"filetype"} = "prefix_only";
}
if (! defined($elem{"fileidx"})) {
$elem{"fileidx"} = 1;
}
&diestack("No result defined") if (! defined($elem{"result"}));
return(\%elem);
}
sub getSearch($)
{
my ($m) = @_;
return($m->{"search"});
}
sub getFileType($)
{
my ($m) = @_;
return($m->{"filetype"});
}
sub getFileIdx($)
{
my ($m) = @_;
return($m->{"fileidx"});
}
sub getExt($)
{
my ($m) = @_;
return($m->{"ext"});
}
sub getResult($)
{
my ($m) = @_;
return($m->{"result"});
}
sub checkForHeader($)
{
my ($l) = @_;
if ($l =~ /^\\begin_header\s*$/) {
my %selem = ();
$selem{type} = "header";
$selem{name} = $1;
unshift(@stack, \%selem);
my @rElems = ();
$rElems[0] = &newMatch("search" => '^\\\\master\s+(.*\.lyx)',
"filetype" => "prefix_only",
"result" => ["\\master ", ""]);
if (keys %{$rFont}) {
for my $ff ( keys %{$rFont}) {
my $elem = &newMatch("search" => '^\\\\font_' . $ff . '\s+default',
"filetype" => "replace_only",
"result" => ["\\font_$ff ", $rFont->{$ff}]);
push(@rElems, $elem);
}
my $elemntf = &newMatch("search" => '^\\\\use_non_tex_fonts\s+false',
"filetype" => "replace_only",
"result" => ["\\use_non_tex_fonts true"]);
push(@rElems, $elemntf);
}
&setMatching(\@rElems);
return(1);
}
return(0);
}
sub checkForPreamble($)
{
my ($l) = @_;
if ($l =~ /^\\begin_preamble\s*$/) {
my %selem = ();
$selem{type} = "preamble";
$selem{name} = $1;
unshift(@stack, \%selem);
my $rElem = &newMatch("ext" => [".eps", ".png"],
"search" => '^\\\\photo(.*\{)(.*)\}',
"fileidx" => 2,
"result" => ["\\photo", "1", "2", "}"]);
&setMatching([$rElem]);
return(1);
}
return(0);
}
sub checkForLayoutStart($)
{
my ($l) = @_;
if ($l =~ /^\\begin_layout\s+(.+)$/) {
#print "started layout\n";
my %selem = ();
$selem{type} = "layout";
$selem{name} = $1;
unshift(@stack, \%selem);
if ($selem{name} eq "Picture") {
my $rElem = &newMatch("ext" => [".eps", ".png"],
"search" => '^(.+)',
"result" => ["", "", ""]);
&setMatching([$rElem]);
}
return(1);
}
return(0);
}
sub checkForInsetStart($)
{
my ($l) = @_;
if ($l =~ /^\\begin_inset\s+(.*)$/) {
#print "started inset\n";
my %selem = ();
$selem{type} = "inset";
$selem{name} = $1;
unshift(@stack, \%selem);
if ($selem{name} =~ /^(Graphics|External)$/) {
my $rElem = &newMatch("search" => '^\s+filename\s+(.+)$',
"filetype" => "copy_only",
"result" => ["\tfilename ", "", ""]);
&setMatching([$rElem]);
}
return(1);
}
return(0);
}
sub checkForLatexCommand($)
{
my ($l) = @_;
if ($stack[0]->{type} eq "inset") {
if ($l =~ /^LatexCommand\s+([^\s]+)\s*$/) {
my $param = $1;
if ($stack[0]->{name} =~ /^CommandInset\s+bibtex$/) {
if ($param eq "bibtex") {
my $rElem1 = &newMatch("ext" => ".bib",
"filetype" => "prefix_for_list",
"search" => '^bibfiles\s+\"(.+)\"',
"result" => ["bibfiles \"", "1", "\""]);
my $rElem2 = &newMatch("ext" => ".bst",
"filetype" => "prefix_for_list",
"search" => '^options\s+\"(.+)\"',
"result" => ["options \"", "1", "\""]);
&setMatching([$rElem1, $rElem2]);
}
}
elsif ($stack[0]->{name} =~ /^CommandInset\s+include$/) {
if ($param =~ /^(verbatiminput\*?|lstinputlisting)$/) {
my $rElem = &newMatch("search" => '^filename\s+\"(.+)\"',
"filetype" => "copy_only",
"result" => ["filename \"", "", "\""]);
&setMatching([$rElem]);
}
elsif ($param =~ /^(include|input)$/) {
my $rElem = &newMatch("search" => '^filename\s+\"(.+)\"',
"filetype" => "interpret",
"result" => ["filename \"", "", "\""]);
&setMatching([$rElem]);
}
}
}
}
return(0);
}
#
# parse the given line
# returns a hash with folloving values
# found: 1 if line matched some regex
# fileidx: index into result
# ext: list of possible extensions to use for a valid file
# filelist: list of found file-pathes (may be more then one, e.g. in bibfiles spec)
# separator: to be used while concatenating the filenames
# filetype: prefix_only,replace_only,copy_only,interpret
# same as before, but without 'prefix_for_list'
sub checkLyxLine($)
{
my ($l) = @_;
return({"found" => 0}) if (&checkForHeader($l));
return({"found" => 0}) if (&checkForPreamble($l));
return({"found" => 0}) if (&checkForEndBlock($l));
return({"found" => 0}) if (&checkForLayoutStart($l));
return({"found" => 0}) if (&checkForInsetStart($l));
return({"found" => 0}) if (&checkForLatexCommand($l));
if (defined($stack[0])) {
my $rMatch = &getMatching();
for my $m ( @{$rMatch}) {
my $search = &getSearch($m);
if ($l =~ /$search/) {
my @matches = ($1, $2, $3, $4);
my $filetype = &getFileType($m);
my @result2 = @{&getResult($m)};
for my $r (@result2) {
if ($r =~ /^\d$/) {
$r = $matches[$r-1];
}
}
if ($filetype eq "replace_only") {
# No filename needed
my %result = ("found" => 1,
"filetype" => $filetype,
"result" => \@result2);
return(\%result);
}
else {
my $fileidx = &getFileIdx($m);
my $filename = $matches[$fileidx-1];
if ($filename !~ /^\.*$/) {
my %result = ("found" => 1,
"fileidx" => $fileidx,
"ext" => &getExt($m),
"result" => \@result2);
if ($filetype eq "prefix_for_list") {
# bibfiles|options in CommandInset bibtex
my @filenames = split(',', $filename);
$result{"separator"} = ",";
$result{"filelist"} = \@filenames;
$result{"filetype"} = "prefix_only";
}
else {
$result{"separator"} = "";
$result{"filelist"} = [$filename];
$result{"filetype"} = $filetype;
}
return(\%result);
}
}
}
}
}
return({"found" => 0});
}
1;

View File

@ -5,10 +5,6 @@ export/doc/id/Intro_pdf5
export/doc/id/Shortcuts_pdf5
export/doc/id/Tutorial_pdf5
export/doc/id/UserGuide_pdf5
export/doc/ru/Intro_pdf5
export/doc/ru/Tutorial_pdf5
export/doc/zh_CN/Intro_pdf5
export/doc/zh_CN/Tutorial_pdf5
export/examples/Literate_pdf
export/examples/Literate_pdf2
export/examples/Literate_pdf5
@ -24,7 +20,6 @@ export/examples/hu/example_lyxified_pdf5
export/examples/hu/example_raw_pdf5
export/examples/hu/splash_pdf5
export/examples/id/splash_pdf5
export/examples/ru/splash_pdf5
# See http://www.lyx.org/trac/ticket/8823
export/examples/ja/lilypond_pdf
export/examples/ja/sweave_pdf

View File

@ -0,0 +1,348 @@
#! /usr/bin/env perl
# -*- mode: perl; -*-
#
# file useSystemFonts.pl
# 1.) Copies lyx-files to another location
# 2.) While copying,
# 2a.) searches for relative references to files and
# replaces them with absolute ones
# 2b.) In order to be able to compile with luatex or xetex
# changes default fonts to use non-tex-fonts instead
#
# Syntax: perl useSystemFonts.pl sourceFile destFile format
# Each param represents a path to a file
# sourceFile: full path to a lyx file
# destFile: destination path
# Each subdocument will be copied into a subdirectory of dirname(destFile)
# format: any string of the form '[a-zA-Z0-9]+', e.g. pdf5
#
# This file is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with this software; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# Copyright (c) 2013 Kornel Benko <kornel@lyx.org>
# (c) 2013 Scott Kostyshak <skotysh@lyx.org>
use strict;
BEGIN {
use File::Spec;
my $p = File::Spec->rel2abs( __FILE__ );
$p =~ s/[\/\\]?[^\/\\]+$//;
unshift(@INC, "$p");
}
use File::Basename;
use File::Path;
use Cwd 'abs_path';
use File::Copy "cp";
use File::Temp qw/ :POSIX /;
use lyxStatus;
# convert lyx file to be compilable with xetex
my ($source, $dest, $format, $rest) = @ARGV;
&diestack("Too many arguments") if (defined($rest));
&diestack("Sourcefilename not defined") if (! defined($source));
&diestack("Destfilename not defined") if (! defined($dest));
&diestack("Format (e.g. pdf4) not defined") if (! defined($format));
$source = &abs_path($source);
$dest = &abs_path($dest);
my %font = ();
if ($source =~ /\/(he|el|ru|uk)\//) {
$font{roman} = "FreeSans";
$font{sans} = "FreeSans";
$font{typewriter} = "FreeSans";
}
elsif ($source =~ /\/fa\//) {
$font{roman} = "FreeFarsi";
$font{sans} = "FreeFarsi";
$font{typewriter} = "FreeFarsi Monospace";
}
elsif ($source =~ /\/zh_CN\//) {
$font{roman} = "FreeSans";
$font{sans} = "WenQuanYi Micro Hei";
$font{typewriter} = "WenQuanYi Micro Hei";
}
else {
# Nothing to do?
}
my $sourcedir = dirname($source);
my $destdir = dirname($dest);
if (! -d $destdir) {
&diestack("could not make dir \"$destdir\"") if (! mkdir $destdir);
}
my $destdirOfSubdocuments;
{
my ($name, $pat, $suffix) = fileparse($source, qr/\.[^.]*/);
my $ext = $format;
if ($source =~ /\/([a-z][a-z](_[A-Z][A-Z])?)\//) {
$ext .= "_$1";
}
else {
$ext .= "_main";
}
$destdirOfSubdocuments = "$destdir/tmp_$ext" . "_$name"; # Global var, something TODO here
}
if(-d $destdirOfSubdocuments) {
rmtree($destdirOfSubdocuments);
}
mkdir($destdirOfSubdocuments); # for possibly included files
my %IncludedFiles = ();
my %type2hash = (
"copy_only" => "copyonly",
"interpret" => "interpret");
&addNewJob($source, $dest, "interpret", {}, \%IncludedFiles);
&copyFoundSubdocuments(\%IncludedFiles);
#&printCopiedDocuments(\%IncludedFiles);
exit(0);
###########################################################
sub printCopiedDocuments($)
{
my ($rFiles) = @_;
for my $k (keys %{$rFiles}) {
my $rJob = $rFiles->{$k};
for my $j ( values %type2hash) {
if (defined($rJob->{$j})) {
print "$j: $k->$rJob->{$j}, " . $rJob->{$j . "copied"} . "\n";
}
}
}
}
sub interpretedCopy($$$$)
{
my ($source, $dest, $destdirOfSubdocuments, $rFiles) = @_;
my $sourcedir = dirname($source);
my $res = 0;
&diestack("could not read \"$source\"") if (!open(FI, $source));
&diestack("could not write \"$dest\"") if (! open(FO, '>', $dest));
&initLyxStack(\%font);
while (my $l = <FI>) {
chomp($l);
my $rStatus = &checkLyxLine($l);
if ($rStatus->{found}) {
my $rF = $rStatus->{result};
if ($rStatus->{"filetype"} eq "replace_only") {
# e.g. if no files involved (font chage etc)
$l = join('', @{$rF});
}
else {
my $filelist = $rStatus->{filelist};
my $fidx = $rStatus->{fileidx};
my $separator = $rStatus->{"separator"};
my $foundrelative = 0;
for my $f (@{$filelist}) {
my @isrel = &isrelative($f,
$sourcedir,
$rStatus->{ext});
if ($isrel[0]) {
$foundrelative = 1;
my $ext = $isrel[1];
if ($rStatus->{"filetype"} eq "prefix_only") {
$f = &getNewNameOf("$sourcedir/$f", $rFiles);
}
else {
my ($newname, $res1);
($newname, $res1) = &addFileCopyJob("$sourcedir/$f$ext",
"$destdirOfSubdocuments",
$rStatus->{"filetype"},
$rFiles);
print "Added ($res1) file \"$sourcedir/$f$ext\" to be copied to \"$newname\"\n";
if ($ext ne "") {
$newname =~ s/$ext$//;
}
$f = $newname;
$res += $res1;
}
}
}
if ($foundrelative) {
$rF->[$fidx] = join($separator, @{$filelist});
$l = join('', @{$rF});
}
}
}
print FO "$l\n";
}
close(FI);
close(FO);
&closeLyxStack();
return($res);
}
sub copyFoundSubdocuments($)
{
my ($rFiles) = @_;
my $res = 0;
do {
$res = 0;
my %copylist = ();
for my $filename (keys %{$rFiles}) {
next if (! &copyJobPending($filename, $rFiles));
$copylist{$filename} = 1;
}
for my $f (keys %copylist) {
# Second loop needed, because here $rFiles may change
my ($res1, @destfiles) = &copyJob($f, $rFiles);
$res += $res1;
for my $destfile (@destfiles) {
print "res1 = $res1 for \"$f\" to be copied to $destfile\n";
}
}
} while($res > 0); # loop, while $rFiles changed
}
sub copyJob($$)
{
my ($source, $rFiles) = @_;
my $sourcedir = dirname($source);
my $res = 0;
my @dest = ();
for my $k (values %type2hash) {
if ($rFiles->{$source}->{$k}) {
if (! $rFiles->{$source}->{$k . "copied"}) {
$rFiles->{$source}->{$k . "copied"} = 1;
my $dest = $rFiles->{$source}->{$k};
push(@dest, $dest);
if ($k eq "copyonly") {
&diestack("Could not copy \"$source\" to \"$dest\"") if (! cp($source, $dest));
}
else {
&interpretedCopy($source, $dest, $destdirOfSubdocuments, $rFiles);
}
$res += 1;
}
}
}
return($res, @dest);
}
# Trivial check
sub isrelativeFix($$$)
{
my ($f, $sourcedir, $ext) = @_;
return(1, $ext) if (-e "$sourcedir/$f$ext");
return(0,0);
}
sub isrelative($$$)
{
my ($f, $sourcedir, $ext) = @_;
if (ref($ext) eq "ARRAY") {
for my $ext2 (@{$ext}) {
my @res = &isrelativeFix($f, $sourcedir, $ext2);
if ($res[0]) {
return(@res);
}
}
return(0,0);
}
else {
return(&isrelativeFix($f, $sourcedir, $ext));
}
}
sub createTemporaryFileName($$)
{
my ($source, $destdir) = @_;
# get the basename to be used for the template
my ($name, $path, $suffix) = fileparse($source, qr/\.[^.]*/);
#print "source = $source, name = $name, path = $path, suffix = $suffix\n";
my $template = "xx_$name" . "_";
my $fname = File::Temp::tempnam($destdir, $template);
# Append extension from source
if ($suffix ne "") {
$fname .= "$suffix";
}
return($fname);
}
# Check, if file not copied yet
sub copyJobPending($$)
{
my ($f, $rFiles) = @_;
for my $t (values %type2hash) {
if (defined($rFiles->{$f}->{$t})) {
return 1 if (! $rFiles->{$f}->{$t . "copied"});
}
}
return 0;
}
sub addNewJob($$$)
{
my ($source, $newname, $hashname, $rJob, $rFiles) = @_;
$rJob->{$hashname} = $newname;
$rJob->{$hashname . "copied"} = 0;
$rFiles->{$source} = $rJob;
}
sub addFileCopyJob($$$$)
{
my ($source, $destdirOfSubdocuments, $filetype, $rFiles) = @_;
my ($res, $newname) = (0, undef);
my $rJob = $rFiles->{$source};
my $hashname = $type2hash{$filetype};
if (! defined($hashname)) {
&diestack("unknown filetype \"$filetype\"");
}
if (!defined($rJob->{$hashname})) {
&addNewJob($source,
&createTemporaryFileName($source, $destdirOfSubdocuments),
"$hashname", $rJob, $rFiles);
$res = 1;
}
$newname = $rJob->{$hashname};
return($newname, $res);
}
sub getNewNameOf($$)
{
my ($f, $rFiles) = @_;
my $resultf = $f;
if (defined($rFiles->{$f})) {
for my $t (values %type2hash) {
if (defined($rFiles->{$f}->{$t})) {
$resultf = $rFiles->{$f}->{$t};
last;
}
}
}
return($resultf);
}