Split pdf format as discussed on the list

Previously, the format used for included pdf files was the same as for
document export via ps2pdf. This caused unwanted conversion routes, e.g.
export via odt->pdf instead of dvi->ps->pdf.
I renamed the format for included graphics and not for exported documents,
since otherwise the command line syntax for export would change. This would
require more adaptions for the users, since with the chosen solution the
custom converters are almost always changed correctly in prefs2prefs(),
so that only custom external templates need manual adjustement.

RELEASE-NOTES

@@ -51,6 +51,13 @@ The following pref variables are obsoleted in 2.1:
 To set the default language and paper size for new documents, use the
 "Save As Document Defaults" button in Document > Settings.
 
+The format pdf aka "PDF (ps2pdf)" was split in two formats: pdf aka
+"PDF (ps2pdf)" and pdf6 aka "PDF (graphics)". pdf6 is used for PDF files
+included as graphics. Custom external templates and converters might need
+to be adjusted. The format split was needed since otherwise an odt->pdf
+converter which is intended for included graphics only did interfere with
+document export via ps2pdf.
+
 The support for ArmTeX (a special typesetting engine for Armenian) was removed.
 Writing text in Armenian is now possible in a native way using XeTeX.
 

lib/configure.py

@@ -579,13 +579,15 @@ def checkFormatEntries(dtl_tools):
 \Format eps2       eps    "EPS (uncropped)"       "" "%%"	""	"vector"	""
 \Format ps         ps      Postscript             t  "%%"	""	"document,vector,menu=export"	"application/postscript"'''])
     # for xdg-open issues look here: http://www.mail-archive.com/lyx-devel@lists.lyx.org/msg151818.html
+    # the MIME type is set for pdf6, because that one needs to be autodetectable by libmime
     checkViewer('a PDF previewer', ['pdfview', 'kpdf', 'okular', 'evince', 'kghostview', 'xpdf', 'acrobat', 'acroread', \
 		    'gv', 'ghostview'],
-        rc_entry = [r'''\Format pdf        pdf    "PDF (ps2pdf)"          P  "%%"	""	"document,vector,menu=export"	"application/pdf"
+        rc_entry = [r'''\Format pdf        pdf    "PDF (ps2pdf)"          P  "%%"	""	"document,vector,menu=export"	""
 \Format pdf2       pdf    "PDF (pdflatex)"        F  "%%"	""	"document,vector,menu=export"	""
 \Format pdf3       pdf    "PDF (dvipdfm)"         m  "%%"	""	"document,vector,menu=export"	""
 \Format pdf4       pdf    "PDF (XeTeX)"           X  "%%"	""	"document,vector,menu=export"	""
-\Format pdf5       pdf    "PDF (LuaTeX)"          u  "%%"	""	"document,vector,menu=export"	""'''])
+\Format pdf5       pdf    "PDF (LuaTeX)"          u  "%%"	""	"document,vector,menu=export"	""
+\Format pdf6       pdf    "PDF (graphics)"        "" "%%"	""	"vector"	"application/pdf"'''])
     #
     checkViewer('a DVI previewer', ['xdvi', 'kdvi', 'okular', 'yap', 'dviout -Set=!m'],
         rc_entry = [r'''\Format dvi        dvi     DVI                    D  "%%"	""	"document,vector,menu=export"	"application/x-dvi"
@@ -730,9 +732,10 @@ def checkConverterEntries():
     #
     checkProg('an OpenDocument -> LaTeX converter', ['w2l -clean $$i'],
         rc_entry = [ r'\converter odt        latex      "%%"	""' ])
-    #
+    # Only define a converter to pdf6, otherwise the odt format could be
+    # used as an intermediate step for export to pdf, which is not wanted.
     checkProg('an OpenDocument -> PDF converter', ['unoconv -f pdf --stdout $$i > $$o'],
-        rc_entry = [ r'\converter odt        pdf        "%%"	""' ])
+        rc_entry = [ r'\converter odt        pdf6       "%%"	""' ])
     # According to http://www.tug.org/applications/tex4ht/mn-commands.html
     # the command mk4ht oolatex $$i has to be used as default,
     # but as this would require to have Perl installed, in MiKTeX oolatex is
@@ -749,7 +752,7 @@ def checkConverterEntries():
     #
     checkProg('a RTF -> HTML converter', ['unrtf --html  $$i > $$o'],
         rc_entry = [ r'\converter rtf      html        "%%"	""' ])
-    #
+    # Do not define a converter to pdf6, ps is a pure export format 
     checkProg('a PS to PDF converter', ['ps2pdf13 $$i $$o'],
         rc_entry = [ r'\converter ps         pdf        "%%"	""' ])
     #
@@ -770,9 +773,9 @@ def checkConverterEntries():
     #
     checkProg('a PDF to PS converter', ['pdf2ps $$i $$o', 'pdftops $$i $$o'],
         rc_entry = [ r'\converter pdf         ps        "%%"	""' ])
-    #
+    # Only define a converter from pdf6 for graphics
     checkProg('a PDF to EPS converter', ['pdftops -eps -f 1 -l 1 $$i $$o'],
-        rc_entry = [ r'\converter pdf         eps        "%%"	""' ])
+        rc_entry = [ r'\converter pdf6        eps        "%%"	""' ])
     #
     checkProg('a Beamer info extractor', ['makebeamerinfo -p $$i'],
         rc_entry = [ r'\converter pdf2         beamer.info        "%%"	""' ])
@@ -804,7 +807,7 @@ def checkConverterEntries():
         rc_entry = [
             r'''\converter tgif       eps        "tgif -print -color -eps -stdout $$i > $$o"	""
 \converter tgif       png        "tgif -print -color -png -o $$d $$i"	""
-\converter tgif       pdf        "tgif -print -color -pdf -stdout $$i > $$o"	""''',
+\converter tgif       pdf6       "tgif -print -color -pdf -stdout $$i > $$o"	""''',
             ''])
     #
     checkProg('a WMF -> EPS converter', ['metafile2eps $$i $$o', 'wmf2eps -o $$o $$i'],
@@ -812,17 +815,17 @@ def checkConverterEntries():
     #
     checkProg('an EMF -> EPS converter', ['metafile2eps $$i $$o', 'wmf2eps -o $$o $$i'],
         rc_entry = [ r'\converter emf        eps        "%%"	""'])
-    #
+    # Only define a converter to pdf6 for graphics
     checkProg('an EPS -> PDF converter', ['epstopdf'],
-        rc_entry = [ r'\converter eps        pdf        "epstopdf --outfile=$$o $$i"	""', ''])
+        rc_entry = [ r'\converter eps        pdf6       "epstopdf --outfile=$$o $$i"	""', ''])
     #
     checkProg('an EPS -> PNG converter', ['convert $$i $$o'],
         rc_entry = [ r'\converter eps        png        "%%"	""', ''])
     #
-    # no agr -> pdf converter, since the pdf library used by gracebat is not
+    # no agr -> pdf6 converter, since the pdf library used by gracebat is not
     # free software and therefore not compiled in in many installations.
     # Fortunately, this is not a big problem, because we will use epstopdf to
-    # convert from agr to pdf via eps without loss of quality.
+    # convert from agr to pdf6 via eps without loss of quality.
     checkProg('a Grace -> Image converter', ['gracebat'],
         rc_entry = [
             r'''\converter agr        eps        "gracebat -hardcopy -printfile $$o -hdevice EPS $$i 2>/dev/null"	""
@@ -848,9 +851,9 @@ def checkConverterEntries():
     # odg->png and odg->pdf converters, since the bb would be too large as well.
     checkProg('an OpenOffice -> EPS converter', ['libreoffice -headless -nologo -convert-to eps $$i', 'unoconv -f eps --stdout $$i > $$o'],
         rc_entry = [ r'\converter odg        eps2       "%%"	""'])
-    #
+    # Only define a converter to pdf6 for graphics
     checkProg('a SVG -> PDF converter', ['rsvg-convert -f pdf -o $$o $$i', 'inkscape --file=$$i --export-area-drawing --without-gui --export-pdf=$$o'],
-        rc_entry = [ r'\converter svg        pdf        "%%"	""'])
+        rc_entry = [ r'\converter svg        pdf6       "%%"	""'])
     #
     checkProg('a SVG -> EPS converter', ['rsvg-convert -f ps -o $$o $$i', 'inkscape --file=$$i --export-area-drawing --without-gui --export-eps=$$o'],
         rc_entry = [ r'\converter svg        eps        "%%"	""'])
@@ -876,13 +879,13 @@ def checkConverterEntries():
             if int(version[0]) > 2 or (len(version) > 1 and int(version[0]) == 2 and int(version[1]) >= 11):
                 addToRC(r'''\converter lilypond   eps        "lilypond -dbackend=eps -dsafe --ps $$i"	""
 \converter lilypond   png        "lilypond -dbackend=eps -dsafe --png $$i"	""''')
-                addToRC(r'\converter lilypond   pdf        "lilypond -dbackend=eps -dsafe --pdf $$i"	""')
+                addToRC(r'\converter lilypond   pdf6       "lilypond -dbackend=eps -dsafe --pdf $$i"	""')
                 logger.info('+  found LilyPond version %s.' % version_number)
             elif int(version[0]) > 2 or (len(version) > 1 and int(version[0]) == 2 and int(version[1]) >= 6):
                 addToRC(r'''\converter lilypond   eps        "lilypond -b eps --ps --safe $$i"	""
 \converter lilypond   png        "lilypond -b eps --png $$i"	""''')
                 if int(version[0]) > 2 or (len(version) > 1 and int(version[0]) == 2 and int(version[1]) >= 9):
-                    addToRC(r'\converter lilypond   pdf        "lilypond -b eps --pdf --safe $$i"	""')
+                    addToRC(r'\converter lilypond   pdf6       "lilypond -b eps --pdf --safe $$i"	""')
                 logger.info('+  found LilyPond version %s.' % version_number)
             else:
                 logger.info('+  found LilyPond, but version %s is too old.' % version_number)
@@ -1387,7 +1390,7 @@ if __name__ == '__main__':
     lyx_check_config = True
     lyx_kpsewhich = True
     outfile = 'lyxrc.defaults'
-    lyxrc_fileformat = 10
+    lyxrc_fileformat = 11
     rc_entries = ''
     lyx_keep_temps = False
     version_suffix = ''

lib/doc/Customization.lyx

@@ -1,5 +1,5 @@
 #LyX 2.1 created this file. For more info see http://www.lyx.org/
-\lyxformat 463
+\lyxformat 467
 \begin_document
 \begin_header
 \textclass scrbook
@@ -101,6 +101,7 @@ End
 \use_geometry false
 \use_package amsmath 0
 \use_package amssymb 0
+\use_package cancel 0
 \use_package esint 0
 \use_package mathdots 1
 \use_package mathtools 0
@@ -2592,8 +2593,13 @@ You do not have to define converters for all formats between which you want
  choose the shortest possible chain.
  You can, though, still define multiple conversion methods between file
  formats.
- For example, the standard LyX configuration provides three ways to convert
+ For example, the standard LyX configuration provides 
- LaTeX to PDF: Directly, using 
+\change_inserted -195340706 1363813100
+five
+\change_deleted -195340706 1363813102
+three
+\change_unchanged
+ ways to convert LaTeX to PDF: Directly, using 
 \begin_inset Flex MenuItem
 status collapsed
 
@@ -2613,7 +2619,11 @@ ps2pdf
 
 \end_inset
 
-; or via DVI, using 
+; 
+\change_deleted -195340706 1363813149
+or 
+\change_unchanged
+via DVI, using 
 \begin_inset Flex MenuItem
 status collapsed
 
@@ -2623,6 +2633,10 @@ dvipdfm
 
 \end_inset
 
+
+\change_inserted -195340706 1363813143
+; or directly using XeTeX or LuaTeX
+\change_unchanged
 .
  To define such alternate chains, you must define multiple target `file
  formats', as described in section 
@@ -2653,12 +2667,46 @@ pdf2
 
 \end_inset
 
+
+\change_inserted -195340706 1363813075
+, 
+\begin_inset Flex MenuItem
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -195340706 1363813079
+pdf3
+\end_layout
+
+\end_inset
+
+, 
+\begin_inset Flex MenuItem
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -195340706 1363813082
+pdf4
+\end_layout
+
+\end_inset
+
+
+\change_unchanged
 , and 
 \begin_inset Flex MenuItem
 status collapsed
 
 \begin_layout Plain Layout
-pdf3
+pdf
+\change_inserted -195340706 1363813086
+5
+\change_deleted -195340706 1363813086
+3
+\change_unchanged
+
 \end_layout
 
 \end_inset
@@ -23545,6 +23593,25 @@ File Format
 
  dialog).
  This command must occur exactly once.
+
+\change_inserted -195340706 1363813305
+ If the resulting file format is PDF, you need to specify the format 
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -195340706 1363813269
+pdf6
+\end_layout
+
+\end_inset
+
+.
+ This is the PDF format used for including graphics.
+ The other defined PDF formats are for document export.
+\change_unchanged
+
 \end_layout
 
 \begin_layout Description

lib/external_templates

@@ -284,7 +284,7 @@ Template LilyPond
 		TransformOption Extra  ExtraOption
 		Option Arg "[$$Extra,$$Rotate,$$Resize,$$Clip]"
 		Product "\\includegraphics$$Arg{$$AbsOrRelPathMaster$$Basename}"
-		UpdateFormat pdf
+		UpdateFormat pdf6
 		UpdateResult "$$AbsPath$$Basename.pdf"
 		Requirement "graphicx"
 		ReferencedFile pdflatex "$$AbsPath$$Basename.pdf"
@@ -313,7 +313,7 @@ Template PDFPages
 		Read the documentation of the pdfpages package
 		for further options and details.
 	HelpTextEnd
-	InputFormat pdf
+	InputFormat pdf6
 	FileFilter "*.pdf"
 	AutomaticProduction true
 	Transform Rotate
@@ -326,7 +326,7 @@ Template PDFPages
 		TransformOption Extra  ExtraOption
 		Option Arg "[$$Extra,$$Rotate,$$Resize]"
 		Product "\\includepdf$$Arg{$$AbsOrRelPathMaster$$Basename}"
-		UpdateFormat pdf
+		UpdateFormat pdf6
 		UpdateResult "$$AbsPath$$Basename.pdf"
 		Requirement "pdfpages"
 		ReferencedFile pdflatex "$$AbsPath$$Basename.pdf"
@@ -407,7 +407,7 @@ Template Dia
 		TransformOption Extra  ExtraOption
 		Option Arg "[$$Extra,$$Rotate,$$Resize,$$Clip]"
 		Product "\\includegraphics$$Arg{$$AbsOrRelPathMaster$$Basename}"
-		UpdateFormat pdf
+		UpdateFormat pdf6
 		UpdateResult "$$AbsPath$$Basename.pdf"
 		Requirement "graphicx"
 		ReferencedFile pdflatex "$$AbsPath$$Basename.pdf"

lib/scripts/prefs2prefs_prefs.py

@@ -50,6 +50,9 @@
 #  Add close_buffer_with_last_view in preferences.
 #  No conversion necessary.
 
+# Incremented to format 11, by gb
+#   Split pdf format into pdf and pdf6
+
 import re
 
 ###########################################################
@@ -236,6 +239,45 @@ def add_mime_types(line):
 		converted = converted + '       ""'
 	return (True, converted)
 
+def split_pdf_format(line):
+	# strictly speaking, a new format would not require to bump the
+	# version number, but the old pdf format was hardcoded at several
+	# places in the C++ code, so an update seemed like a good idea.
+	if line.lower().startswith("\\format"):
+		entries = get_format(line)
+		if entries[1] == 'pdf':
+			if len(entries) < 6:
+				viewer = ''
+			else:
+				viewer = entries[5]
+			converted = line.replace('application/pdf', '') + '''
+\Format pdf6       pdf    "PDF (graphics)"        "" "''' + viewer + '"	""	"vector"	"application/pdf"'
+			return (True, converted)
+	elif line.lower().startswith("\\viewer_alternatives") or \
+	     line.lower().startswith("\\editor_alternatives"):
+		entries = get_format(line)
+		if entries[1] == 'pdf':
+			converted = line + "\n" + entries[0] + ' pdf6 "' + entries[2] + '"'
+			return (True, converted)
+	elif line.lower().startswith("\\converter"):
+		entries = get_format(line)
+		# The only converter from pdf that is touched is pdf->eps:
+		# All other converters are likely meant for further processing on export.
+		# The only converter to pdf that stays untouched is dvi->pdf:
+		# All other converters are likely meant for graphics.
+		if (entries[1] == 'pdf' and entries[2] == 'eps') or \
+		   (entries[1] != 'ps'  and entries[2] == 'pdf'):
+			if entries[1] == 'pdf':
+				converted = entries[0] + ' pdf6 ' + entries[2]
+			else:
+				converted = entries[0] + ' ' + entries[1] + ' pdf6'
+			i = 3
+			while i < len(entries):
+				converted = converted + ' "' + entries[i] + '"'
+				i = i + 1
+			return (True, converted)
+	return no_match
+
 def remove_default_language(line):
 	if not line.lower().startswith("\\default_language"):
 		return no_match
@@ -262,5 +304,6 @@ conversions = [
 	[ 7, [add_mime_types]],
 	[ 8, []],
 	[ 9, [ remove_default_language ]],
-	[ 10, []]
+	[ 10, []],
+	[ 11, [split_pdf_format]]
 ]

src/ConverterCache.cpp

@@ -271,7 +271,7 @@ void ConverterCache::add(FileName const & orig_from, string const & to_format,
 		add(orig_from, "eps", converted_eps);
 	} else if (to_format == "pdftex") {
 		FileName const converted_pdf(changeExtension(converted_file.absFileName(), "pdf"));
-		add(orig_from, "pdf", converted_pdf);
+		add(orig_from, "pdf6", converted_pdf);
 	}
 
 	// Is the file in the cache already?
@@ -396,6 +396,20 @@ bool ConverterCache::inCache(FileName const & orig_from,
 		LYXERR(Debug::FILES, "not in cache.");
 		return false;
 	}
+
+	// Special handling of pstex and pdftex formats: These are only
+	// considered to be in the cache if the corresponding graphics
+	// fiels are there as well. Otherwise copy() of the graphics below
+	// would fail.
+	// FIXME: Should not hardcode this (see bug 3819 for details)
+	if (to_format == "pstex") {
+		if (!inCache(orig_from, "eps"))
+			return false;
+	} else if (to_format == "pdftex") {
+		if (!inCache(orig_from, "pdf6"))
+			return false;
+	}
+
 	time_t const timestamp = orig_from.lastModified();
 	if (item->timestamp == timestamp) {
 		LYXERR(Debug::FILES, "identical timestamp.");
@@ -436,7 +450,7 @@ bool ConverterCache::copy(FileName const & orig_from, string const & to_format,
 			return false;
 	} else if (to_format == "pdftex") {
 		FileName const dest_pdf(changeExtension(dest.absFileName(), "pdf"));
-		if (!copy(orig_from, "pdf", dest_pdf))
+		if (!copy(orig_from, "pdf6", dest_pdf))
 			return false;
 	}
 

src/Format.cpp

@@ -336,7 +336,8 @@ string guessFormatFromContents(FileName const & fn)
 			format = "jpg";
 
 		else if (contains(str, "%PDF"))
-			format = "pdf";
+			// autodetect pdf format for graphics inclusion
+			format = "pdf6";
 
 		else if (contains(str, "PNG"))
 			format = "png";
@@ -681,7 +682,7 @@ bool Formats::edit(Buffer const & buffer, FileName const & filename,
 
 	// LinkBack files look like PDF, but have the .linkback extension
 	string const ext = getExtension(filename.absFileName());
-	if (format_name == "pdf" && ext == "linkback") {
+	if (format_name == "pdf6" && ext == "linkback") {
 #ifdef USE_MACOSX_PACKAGING
 		return editLinkBackFile(filename.absFileName().c_str());
 #else

src/LyXRC.cpp

@@ -55,7 +55,7 @@ namespace os = support::os;
 
 namespace {
 
-static unsigned int const LYXRC_FILEFORMAT = 10; // tommaso: new prefs option: close_buffer_with_last_view
+static unsigned int const LYXRC_FILEFORMAT = 11; // gb: Split pdf format into pdf and pdf6
 
 // when adding something to this array keep it sorted!
 LexerKeyword lyxrcTags[] = {

src/insets/InsetGraphics.cpp

@@ -110,7 +110,7 @@ string findTargetFormat(string const & format, OutputParams const & runparams)
 		Format const * const f = formats.getFormat(format);
 		// Convert vector graphics to pdf
 		if (f && f->vectorFormat())
-			return "pdf";
+			return "pdf6";
 		// pdflatex can use jpeg, png and pdf directly
 		if (format == "jpg")
 			return format;