Implement forced local layouts

These should be used if any new style needs to be introduced in the stable
2.1 series: If the ForceLocal flag of the style is set, it will always be
written to the document header, so that even older 2.1 versions can read
and correctly output the document.

development/FORMAT

@@ -11,6 +11,11 @@ adjustments are made to tex2lyx and bugs are fixed in lyx2lyx.
 
 -----------------------
 
+2013-05-15 Georg Baum  <Georg.Baum@post.rwth-aachen.de>
+	* Format incremented to 470
+	  forced local layouts for future layout backward compatibility:
+	  \begin_forced_local_layout, \end_forced_local_layout
+
 2013-03-23 Georg Baum  <Georg.Baum@post.rwth-aachen.de>
 	* Format incremented to 469
 	  support for \caption* in longtables (fix bug 3209)

lib/doc/Customization.lyx

@@ -1,5 +1,5 @@
 #LyX 2.1 created this file. For more info see http://www.lyx.org/
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass scrbook
@@ -11901,6 +11901,77 @@ LabelFont
 \end_inset
 
 .
+\change_inserted -195340706 1364753581
+
+\end_layout
+
+\begin_layout Description
+
+\change_inserted -195340706 1364753581
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -195340706 1364753581
+ForceLocal n
+\end_layout
+
+\end_inset
+
+ Used for backporting new styles to stable LyX versions.
+ The first stable version that supports this tag is LyX 2.1.0.
+ 
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -195340706 1364753581
+n
+\end_layout
+
+\end_inset
+
+ is a number which may either be 0 (this is the default if the tag is not
+ given), -1 or any value greater than zero.
+ If the 
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -195340706 1364753581
+ForceLocal
+\end_layout
+
+\end_inset
+
+ flag of a style is greater than zero, it will always be written to the
+ document header.
+ If a .lyx file is read, the style definitions from the document header are
+ added to the document class.
+ Therefore even older versions can handle the style.
+ The argument of 
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -195340706 1364753581
+ForceLocal
+\end_layout
+
+\end_inset
+
+ is a version number: If such a style is read, and the version number is
+ less than the version number of the already existing style in the document
+ class, the new style is ignored.
+ If the version number is higher, the new style replaces the existing style.
+ A value of -1 means an infinite version number, i.e.
+ it is always used.
+\change_unchanged
+
 \end_layout
 
 \begin_layout Description

lib/doc/Development.lyx

@@ -1,5 +1,5 @@
 #LyX 2.1 created this file. For more info see http://www.lyx.org/
-\lyxformat 462
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass scrbook
@@ -68,6 +68,7 @@ End
 \use_geometry false
 \use_package amsmath 1
 \use_package amssymb 1
+\use_package cancel 0
 \use_package esint 0
 \use_package mathdots 1
 \use_package mathtools 0
@@ -435,6 +436,95 @@ src/tex2lyx/TODO.txt
  recommendation were related to mixed version syntax, not ERT.
 \end_layout
 
+\begin_layout Section
+Backporting new styles to the stable version
+\end_layout
+
+\begin_layout Standard
+Starting with the stable LyX 2.1 branch, there is a mechanism in place to
+ backport new styles to the stable version without the need to update the
+ file format.
+ The basic idea is that the new style definition is automatically copied
+ to the document preamble, so that it can even be used by older minor revisions
+ that did not yet include the style.
+ To backport a new style to the stable version, the following steps are
+ needed:
+\end_layout
+
+\begin_layout Enumerate
+Add the line 
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+ForceLocal -1
+\end_layout
+
+\end_inset
+
+ to the style definition in the development version.
+\end_layout
+
+\begin_layout Enumerate
+Copy the style definition to the stable version, but use 
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+ForceLocal 1
+\end_layout
+
+\end_inset
+
+ instead.
+ If needed adjust the format to the one used by the stable version (see
+ the customization manual for details of the layout file format).
+\end_layout
+
+\begin_layout Enumerate
+For each update of the style in a later stable version, increase the argument
+ of 
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+ForceLocal
+\end_layout
+
+\end_inset
+
+ by one (in the stable version, the development version should not be touched).
+\end_layout
+
+\begin_layout Standard
+For details about the 
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+ForceLocal
+\end_layout
+
+\end_inset
+
+ flag see the customization manual.
+ No 
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+lyx2lyx
+\end_layout
+
+\end_inset
+
+ support is needed for backported styles: Since the style of the development
+ version has an infinite version number, it will always be used.
+ Furthermore, since its version number is less than one, the style will
+ not be written anymore to the document header for files saved by the new
+ version.
+\end_layout
+
 \begin_layout Chapter
 Tests
 \end_layout

lib/lyx2lyx/lyx_2_1.py

@@ -4093,6 +4093,40 @@ def revert_starred_caption(document):
       i = i + 1
 
 
+def revert_forced_local_layout(document):
+    i = 0
+    while True:
+        i = find_token(document.header, "\\begin_forced_local_layout", i)
+        if i == -1:
+            return
+        j = find_end_of(document.header, i, "\\begin_forced_local_layout", "\\end_forced_local_layout")
+        if j == -1:
+            # this should not happen
+            break
+        regexp = re.compile(r'\s*forcelocal', re.IGNORECASE)
+        k = find_re(document.header, regexp, i, j)
+        while k != -1:
+            del document.header[k]
+            j = j - 1
+            k = find_re(document.header, regexp, i, j)
+        k = find_token(document.header, "\\begin_local_layout", 0)
+        if k == -1:
+            document.header[i] = "\\begin_local_layout"
+            document.header[j] = "\\end_local_layout"
+        else:
+            l = find_end_of(document.header, k, "\\begin_local_layout", "\\end_local_layout")
+            if j == -1:
+                # this should not happen
+                break
+            lines = document.header[i+1 : j]
+            if k > i:
+                document.header[k+1 : k+1] = lines
+                document.header[i   : j  ] = []
+            else:
+                document.header[i   : j  ] = []
+                document.header[k+1 : k+1] = lines
+
+
 ##
 # Conversion hub
 #
@@ -4154,10 +4188,12 @@ convert = [
            [466, []],
            [467, []],
            [468, []],
-           [469, []]
+           [469, []],
+           [470, []] 
           ]
 
 revert =  [
+           [469, [revert_forced_local_layout]],
            [468, [revert_starred_caption]],
            [467, [revert_mbox_fbox]],
            [466, [revert_iwona_fonts]],

lib/scripts/layout2layout.py

@@ -154,6 +154,9 @@ import os, re, string, sys
 # Incremented to format 45, 12 February 2013 by rgh
 # New Tag "NoInsetLayout"
 
+# Incremented to format 46, 15 May 2013 by gb
+# New Tag "ForceLocal"
+
 # Do not forget to document format change in Customization
 # Manual (section "Declaring a new text class").
 
@@ -161,7 +164,7 @@ import os, re, string, sys
 # development/tools/updatelayouts.sh script to update all
 # layout files to the new format.
 
-currentFormat = 45
+currentFormat = 46
 
 
 def usage(prog_name):
@@ -355,7 +358,7 @@ def convert(lines):
                     # nothing to do
                     return format
                 else:
-                    error('Cannot convert file format %s' % format)
+                    error('Cannot convert file format %s to %s' % (format, currentFormat))
             else:
                 lines.insert(i, "Format 2")
                 only_comment = 0
@@ -378,7 +381,7 @@ def convert(lines):
                 i += 1
             continue
 
-        if format == 44:
+        if format == 44 or format == 45:
             # nothing to do.
             i += 1
             continue

src/Buffer.cpp

@@ -836,7 +836,8 @@ int Buffer::readHeader(Lexer & lex)
 	params().html_latex_end.clear();
 	params().html_math_img_scale = 1.0;
 	params().output_sync_macro.erase();
-	params().local_layout.clear();
+	params().setLocalLayout(string(), false);
+	params().setLocalLayout(string(), true);
 
 	for (int i = 0; i < 4; ++i) {
 		params().user_defined_bullet(i) = ITEMIZE_DEFAULTS[i];

src/BufferParams.cpp

@@ -627,7 +627,9 @@ string BufferParams::readToken(Lexer & lex, string const & token,
 	} else if (token == "\\begin_preamble") {
 		readPreamble(lex);
 	} else if (token == "\\begin_local_layout") {
-		readLocalLayout(lex);
+		readLocalLayout(lex, false);
+	} else if (token == "\\begin_forced_local_layout") {
+		readLocalLayout(lex, true);
 	} else if (token == "\\begin_modules") {
 		readModules(lex);
 	} else if (token == "\\begin_removed_modules") {
@@ -990,6 +992,7 @@ void BufferParams::writeFile(ostream & os) const
 	   << convert<string>(maintain_unincluded_children) << '\n';
 
 	// local layout information
+	string const local_layout = getLocalLayout(false);
 	if (!local_layout.empty()) {
 		// remove '\n' from the end
 		string const tmplocal = rtrim(local_layout, "\n");
@@ -997,6 +1000,14 @@ void BufferParams::writeFile(ostream & os) const
 		   << tmplocal
 		   << "\n\\end_local_layout\n";
 	}
+	string const forced_local_layout = getLocalLayout(true);
+	if (!forced_local_layout.empty()) {
+		// remove '\n' from the end
+		string const tmplocal = rtrim(forced_local_layout, "\n");
+		os << "\\begin_forced_local_layout\n"
+		   << tmplocal
+		   << "\n\\end_forced_local_layout\n";
+	}
 
 	// then the text parameters
 	if (language != ignore_language)
@@ -2102,14 +2113,16 @@ void BufferParams::makeDocumentClass()
 
 	doc_class_ = getDocumentClass(*baseClass(), mods);
 
-	if (!local_layout.empty()) {
-		TextClass::ReturnValues success =
-			doc_class_->read(local_layout, TextClass::MODULE);
+	TextClass::ReturnValues success = TextClass::OK;
+	if (!forced_local_layout_.empty())
+		success = doc_class_->read(forced_local_layout_, TextClass::MODULE);
+	if (!local_layout_.empty() &&
+	    (success == TextClass::OK || success == TextClass::OK_OLDFORMAT))
+		success = doc_class_->read(local_layout_, TextClass::MODULE);
 	if (success != TextClass::OK && success != TextClass::OK_OLDFORMAT) {
 		docstring const msg = _("Error reading internal layout information");
 		frontend::Alert::warning(_("Read Error"), msg);
 	}
-	}
 }
 
 
@@ -2125,6 +2138,24 @@ bool BufferParams::citationModuleCanBeAdded(string const & modName) const
 }
 
 
+std::string BufferParams::getLocalLayout(bool forced) const
+{
+	if (forced)
+		return doc_class_->forcedLayouts();
+	else
+		return local_layout_;
+}
+
+
+void BufferParams::setLocalLayout(string const & layout, bool forced)
+{
+	if (forced)
+		forced_local_layout_ = layout;
+	else
+		local_layout_ = layout;
+}
+
+
 bool BufferParams::addLayoutModule(string const & modName)
 {
 	LayoutModuleList::const_iterator it = layout_modules_.begin();
@@ -2342,13 +2373,19 @@ void BufferParams::readPreamble(Lexer & lex)
 }
 
 
-void BufferParams::readLocalLayout(Lexer & lex)
+void BufferParams::readLocalLayout(Lexer & lex, bool forced)
 {
-	if (lex.getString() != "\\begin_local_layout")
+	string const expected = forced ? "\\begin_forced_local_layout" :
+	                                 "\\begin_local_layout";
+	if (lex.getString() != expected)
 		lyxerr << "Error (BufferParams::readLocalLayout):"
 			"consistency check failed." << endl;
 
-	local_layout = lex.getLongString("\\end_local_layout");
+	if (forced)
+		forced_local_layout_ =
+			lex.getLongString("\\end_forced_local_layout");
+	else
+		local_layout_ = lex.getLongString("\\end_local_layout");
 }
 
 

src/BufferParams.h

@@ -162,6 +162,10 @@ public:
 	void clearLayoutModules() { layout_modules_.clear(); }
 	/// Clear the removed module list
 	void clearRemovedModules() { removed_modules_.clear(); }
+	/// Get the local layouts
+	std::string getLocalLayout(bool) const;
+	/// Set the local layouts
+	void setLocalLayout(std::string const &, bool);
 
 	/// returns \c true if the buffer contains a LaTeX document
 	bool isLatex() const;
@@ -305,8 +309,6 @@ public:
 	///
 	std::string preamble;
 	///
-	std::string local_layout;
-	///
 	std::string options;
 	/// use the class options defined in the layout?
 	bool use_default_options;
@@ -483,7 +485,7 @@ private:
 	///
 	void readPreamble(Lexer &);
 	///
-	void readLocalLayout(Lexer &);
+	void readLocalLayout(Lexer &, bool);
 	///
 	void readLanguage(Lexer &);
 	///
@@ -513,6 +515,10 @@ private:
 	/// this is for modules that are required by the document class but that
 	/// the user has chosen not to use
 	std::list<std::string> removed_modules_;
+	/// The local layouts without the forced ones
+	std::string local_layout_;
+	/// Forced local layouts only for reading (use getLocalLayout() instead)
+	std::string forced_local_layout_;
 
 	/// the list of included children (for includeonly)
 	std::list<std::string> included_children_;

src/Layout.cpp

@@ -104,6 +104,7 @@ enum LayoutTags {
 	LT_REFPREFIX,
 	LT_RESETARGS,
 	LT_RIGHTDELIM,
+	LT_FORCELOCAL,
 	LT_INTITLE // keep this last!
 };
 
@@ -145,11 +146,32 @@ Layout::Layout()
 	htmlforcecss_ = false;
 	htmltitle_ = false;
 	spellcheck = true;
+	forcelocal = 0;
 	itemcommand_ = "item";
 }
 
 
 bool Layout::read(Lexer & lex, TextClass const & tclass)
+{
+	// If this is an empty layout, or if no force local version is set,
+	// we know that we will not discard the stuff to read
+	if (forcelocal == 0)
+		return readIgnoreForcelocal(lex, tclass);
+	Layout tmp(*this);
+	tmp.forcelocal = 0;
+	bool const ret = tmp.readIgnoreForcelocal(lex, tclass);
+	// Keep the stuff if
+	// - the read version is higher
+	// - both versions are infinity (arbitrary decision)
+	// - the file did not contain any local version (needed for not
+	//   skipping user defined local layouts)
+	if (tmp.forcelocal <= 0 || tmp.forcelocal > forcelocal)
+		*this = tmp;
+	return ret;
+}
+
+
+bool Layout::readIgnoreForcelocal(Lexer & lex, TextClass const & tclass)
 {
 	// This table is sorted alphabetically [asierra 30March96]
 	LexerKeyword layoutTags[] = {
@@ -166,6 +188,7 @@ bool Layout::read(Lexer & lex, TextClass const & tclass)
 		{ "endlabelstring", LT_ENDLABELSTRING },
 		{ "endlabeltype",   LT_ENDLABELTYPE },
 		{ "font",           LT_FONT },
+		{ "forcelocal",     LT_FORCELOCAL },
 		{ "freespacing",    LT_FREE_SPACING },
 		{ "htmlattr",       LT_HTMLATTR },
 		{ "htmlforcecss",   LT_HTMLFORCECSS },
@@ -582,6 +605,10 @@ bool Layout::read(Lexer & lex, TextClass const & tclass)
 		case LT_SPELLCHECK:
 			lex >> spellcheck;
 			break;
+
+		case LT_FORCELOCAL:
+			lex >> forcelocal;
+			break;
 		}
 	}
 	lex.popTable();
@@ -1290,6 +1317,7 @@ void Layout::write(ostream & os) const
 		   << "\n\tEndPreamble\n";
 	os << "\tHTMLTitle " << htmltitle_ << "\n"
 	      "\tSpellcheck " << spellcheck << "\n"
+	      "\tForceLocal " << forcelocal << "\n"
 	      "End\n";
 }
 

src/Layout.h

@@ -310,9 +310,23 @@ public:
 
 	/// Is this spellchecked?
 	bool spellcheck;
+	/**
+	 * Should this layout definition always be written to the document preamble?
+	 * Possible values are:
+	 *   0: Do not enforce local layout
+	 * >=1: Enforce local layout with version forcelocal
+	 *  -1: Enforce local layout with infinite version
+	 * On reading, the forced local layout is only used if its version
+	 * number is greater than the version number of the same layout in the
+	 * document class. Otherwise, it is ignored.
+	 */
+	int forcelocal;
 
 
 private:
+	/// Reads a layout definition from file
+	/// \return true on success.
+	bool readIgnoreForcelocal(Lexer &, TextClass const &);
 	/// generates the default CSS for this layout
 	void makeDefaultCSS() const;
 	///

src/TextClass.cpp

@@ -61,7 +61,7 @@ namespace lyx {
 // development/tools/updatelayouts.sh script, to update the format of
 // all of our layout files.
 //
-int const LAYOUT_FORMAT = 45; // rgh: New Tag "NoInsetLayout"
+int const LAYOUT_FORMAT = 46; // gb: New Tag "ForceLocal"
 
 namespace {
 
@@ -1376,6 +1376,24 @@ bool DocumentClass::addLayoutIfNeeded(docstring const & n) const
 }
 
 
+string DocumentClass::forcedLayouts() const
+{
+	ostringstream os;
+	bool first = true;
+	const_iterator const e = end();
+	for (const_iterator i = begin(); i != e; ++i) {
+		if (i->forcelocal > 0) {
+			if (first) {
+				os << "Format " << LAYOUT_FORMAT << '\n';
+				first = false;
+			}
+			i->write(os);
+		}
+	}
+	return os.str();
+}
+
+
 InsetLayout const & DocumentClass::insetLayout(docstring const & name) const
 {
 	// FIXME The fix for the InsetLayout part of 4812 would be here:

src/TextClass.h

@@ -399,6 +399,8 @@ public:
 	/// add a new layout \c name if it does not exist in layoutlist_
 	/// \return whether we had to add one.
 	bool addLayoutIfNeeded(docstring const & name) const;
+	/// Forced layouts in layout file syntax
+	std::string forcedLayouts() const;
 
 	///////////////////////////////////////////////////////////////////
 	// accessors

src/frontends/qt4/GuiDocument.cpp

@@ -517,7 +517,7 @@ LocalLayout::LocalLayout() : current_id_(0), validated_(false)
 
 void LocalLayout::update(BufferParams const & params, BufferId id)
 {
-	QString layout = toqstr(params.local_layout);
+	QString layout = toqstr(params.getLocalLayout(false));
 	// Nothing to do if the params and preamble are unchanged.
 	if (id == current_id_
 		&& layout == locallayoutTE->document()->toPlainText())
@@ -533,7 +533,7 @@ void LocalLayout::update(BufferParams const & params, BufferId id)
 void LocalLayout::apply(BufferParams & params)
 {
 	string const layout = fromqstr(locallayoutTE->document()->toPlainText());
-	params.local_layout = layout;
+	params.setLocalLayout(layout, false);
 }
 
 

src/tex2lyx/test/CJK.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/CJKutf8.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/Dummy Document.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/DummyDocument.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/Dummy~Document.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/XeTeX-polyglossia.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/box-color-size-space-align.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/test-insets.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/test-modules.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass amsart

src/tex2lyx/test/test-refstyle-theorems.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass book

src/tex2lyx/test/test-structure.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/test.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/tex2lyx/test/verbatim.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.1.0dev
-\lyxformat 469
+\lyxformat 470
 \begin_document
 \begin_header
 \textclass article

src/version.h

@@ -30,8 +30,8 @@ extern char const * const lyx_version_info;
 
 // Do not remove the comment below, so we get merge conflict in
 // independent branches. Instead add your own.
-#define LYX_FORMAT_LYX 469 // gb: \caption*{}
-#define LYX_FORMAT_TEX2LYX 469 // gb: \caption*{}
+#define LYX_FORMAT_LYX 470 // gb: new tag begin_forced_local_layout/end_forced_local_layout
+#define LYX_FORMAT_TEX2LYX 470 // gb: new tag begin_forced_local_layout/end_forced_local_layout
 
 #if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX
 #ifndef _MSC_VER