Mass backport tex2lyx bug fixes.

tex2lyx is no identical with the version in trunk (except for cosmetic changes
and file formats > 413).
The output of the test cases is either unchanged or improved.


git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/branches/BRANCH_2_0_X@40364 a592a061-630c-0410-9148-cb99ea01b6c8

development/cmake/src/tex2lyx/CMakeLists.txt

@@ -12,8 +12,8 @@ project(${_tex2lyx})
 set(LINKED_sources ${TOP_SRC_DIR}/src/lengthcommon.cpp)
 set(LINKED_headers)
 
-foreach(_src insets/InsetLayout Color Counters
+foreach(_src insets/InsetLayout Author Color Counters
-	Encoding FloatList Floating FontInfo
+	Encoding FloatList Floating FontInfo LaTeXPackages
 	Layout LayoutFile LayoutModuleList Lexer ModuleList TextClass
 	Spacing version)
 	list(APPEND LINKED_sources ${TOP_SRC_DIR}/src/${_src}.cpp)

development/scons/scons_manifest.py

@@ -598,6 +598,7 @@ src_mathed_extra_files = Split('''
 src_tex2lyx_header_files = Split('''
     Context.h
     Parser.h
+    Preamble.h
     tex2lyx.h
 ''')
 
@@ -607,7 +608,7 @@ src_tex2lyx_files = Split('''
     Context.cpp
     math.cpp
     Parser.cpp
-    preamble.cpp
+    Preamble.cpp
     table.cpp
     tex2lyx.cpp
     text.cpp
@@ -621,12 +622,14 @@ src_tex2lyx_copied_header_files = Split('''
 
 
 src_tex2lyx_copied_files = Split('''
+    Author.cpp
     Color.cpp
     Counters.cpp
     Encoding.cpp
     FloatList.cpp
     Floating.cpp
     FontInfo.cpp
+    LaTeXPackages.cpp
     Layout.cpp
     LayoutFile.cpp
     LayoutModuleList.cpp

lib/syntax.default

@@ -370,6 +370,9 @@ $$
 % is some code that may occur in a .tex file created by LyX. The re-import
 % works only because the first argument of \texorpdfstring is specified as
 % translatable in this file.
+% If a command puts the contents of an argument inside an own group, use
+% "group" instead of "translate". Otherwise things like font changes would
+% survive the end of the group in LyX (bug 3036).
 
 \abstractname
 \Acrobatmenu{}{}         % from the hyperref package
@@ -515,8 +518,8 @@ $$
 \makelabels
 \maketitle
 \MakeShortVerb{} % from doc.sty, argument must be verbatim
-\markboth{}{translate}
+\markboth{group}{group}
-\markright{translate}
+\markright{group}
 \mathversion{}
 \mbox{translate}
 \mddefault
@@ -703,30 +706,30 @@ thebibliography{}
 
 % Environments that start math mode.
 % $...$, $$...$$, \(...\) and \[...\] are hardcoded in tex2lyx.
-% The arguments are currently ignored.
+% The arguments are currently ignored (apart from displaymath).
 \begin{mathenvironments}
-equation
+equation{displaymath}
-equation*
+equation*{displaymath}
-eqnarray
+eqnarray{displaymath}
-eqnarray*
+eqnarray*{displaymath}
-align
+align{displaymath}
-align*
+align*{displaymath}
-gather
+gather{displaymath}
-gather*
+gather*{displaymath}
-multline
+multline{displaymath}
-multline*
+multline*{displaymath}
-math
+math{}
-displaymath
+displaymath{displaymath}
-flalign
+flalign{displaymath}
-flalign
+flalign{displaymath}
 % These require extra args
-alignat
+alignat{}{displaymath}
-alignat*
+alignat*{displaymath}
-xalignat
+xalignat{}{displaymath}
-xalignat*
+xalignat*{}{displaymath}
-xxalignat
+xxalignat{}{displaymath}
 % These are not known by LyX but work nevertheless:
-empheq
+empheq[]{}{displaymath}
 \end{mathenvironments}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

src/Changes.h

@@ -41,7 +41,7 @@ public:
 		DELETED // deleted text
 	};
 
-	explicit Change(Type t = UNCHANGED, int a = 0, time_t ct = current_time())
+	explicit Change(Type t = UNCHANGED, int a = 0, time_t ct = support::current_time())
 		: type(t), author(a), changetime(ct) {}
 
 	/// is the change similar to the given change such that both can be merged?

src/LaTeXFeatures.cpp

@@ -24,6 +24,7 @@
 #include "Floating.h"
 #include "FloatList.h"
 #include "Language.h"
+#include "LaTeXPackages.h"
 #include "Layout.h"
 #include "Lexer.h"
 #include "LyXRC.h"
@@ -281,8 +282,6 @@ static docstring const lyxref_def = from_ascii(
 //
 /////////////////////////////////////////////////////////////////////
 
-LaTeXFeatures::Packages LaTeXFeatures::packages_;
-
 
 LaTeXFeatures::LaTeXFeatures(Buffer const & b, BufferParams const & p,
 			     OutputParams const & r)
@@ -334,36 +333,6 @@ void LaTeXFeatures::require(set<string> const & names)
 }
 
 
-void LaTeXFeatures::getAvailable()
-{
-	Lexer lex;
-	support::FileName const real_file = libFileSearch("", "packages.lst");
-
-	if (real_file.empty())
-		return;
-
-	lex.setFile(real_file);
-
-	if (!lex.isOK())
-		return;
-
-	// Make sure that we are clean
-	packages_.clear();
-
-	bool finished = false;
-	// Parse config-file
-	while (lex.isOK() && !finished) {
-		switch (lex.lex()) {
-		case Lexer::LEX_FEOF:
-			finished = true;
-			break;
-		default:
-			packages_.insert(lex.getString());
-		}
-	}
-}
-
-
 void LaTeXFeatures::useLayout(docstring const & layoutname)
 {
 	// Some code to avoid loops in dependency definition
@@ -441,13 +410,7 @@ bool LaTeXFeatures::isAvailable(string const & name)
 		//LYXERR0("from=[" << from << "] to=[" << to << "]");
 		return theConverters().isReachable(from, to);
 	}
-
+	return LaTeXPackages::isAvailable(name);
-	if (packages_.empty())
-		getAvailable();
-	string n = name;
-	if (suffixIs(n, ".sty"))
-		n.erase(name.length() - 4);
-	return packages_.find(n) != packages_.end();
 }
 
 

src/LaTeXFeatures.h

@@ -86,8 +86,6 @@ public:
 	void require(std::string const & name);
 	/// Add a set of feature names requirements
 	void require(std::set<std::string> const & names);
-	/// Which of the required packages are installed?
-	static void getAvailable();
 	/// Is the (required) package available?
 	static bool isAvailable(std::string const & name);
 	/// Has the package been required?
@@ -149,10 +147,6 @@ private:
 	typedef std::list<std::string> SnippetList;
 	///
 	SnippetList preamble_snippets_;
-	/// The available (required) packages
-	typedef std::set<std::string> Packages;
-	///
-	static Packages packages_;
 	///
 	typedef std::set<Language const *> LanguageList;
 	/// used languages (only those that are supported by babel)

src/LaTeXPackages.cpp

@@ -0,0 +1,75 @@
+/**
+ * \file LaTeXPackages.cpp
+ * This file is part of LyX, the document processor.
+ * Licence details can be found in the file COPYING.
+ *
+ * \author José Matos
+ * \author Lars Gullik Bjønnes
+ * \author Jean-Marc Lasgouttes
+ * \author Jürgen Vigna
+ * \author André Pönitz
+ *
+ * Full author contact details are available in file CREDITS.
+ */
+
+#include <config.h>
+
+#include "LaTeXPackages.h"
+
+#include "Lexer.h"
+
+#include "support/FileName.h"
+#include "support/filetools.h"
+#include "support/lstrings.h"
+
+
+using namespace std;
+using namespace lyx::support;
+
+
+namespace lyx {
+
+LaTeXPackages::Packages LaTeXPackages::packages_;
+
+
+void LaTeXPackages::getAvailable()
+{
+	Lexer lex;
+	support::FileName const real_file = libFileSearch("", "packages.lst");
+
+	if (real_file.empty())
+		return;
+
+	lex.setFile(real_file);
+
+	if (!lex.isOK())
+		return;
+
+	// Make sure that we are clean
+	packages_.clear();
+
+	bool finished = false;
+	// Parse config-file
+	while (lex.isOK() && !finished) {
+		switch (lex.lex()) {
+		case Lexer::LEX_FEOF:
+			finished = true;
+			break;
+		default:
+			packages_.insert(lex.getString());
+		}
+	}
+}
+
+
+bool LaTeXPackages::isAvailable(string const & name)
+{
+	if (packages_.empty())
+		getAvailable();
+	string n = name;
+	if (suffixIs(n, ".sty"))
+		n.erase(name.length() - 4);
+	return packages_.find(n) != packages_.end();
+}
+
+} // namespace lyx

src/LaTeXPackages.h

@@ -0,0 +1,41 @@
+// -*- C++ -*-
+/**
+ * \file LaTeXPackages.h
+ * This file is part of LyX, the document processor.
+ * Licence details can be found in the file COPYING.
+ *
+ * \author Lars Gullik Bjønnes
+ * \author Jean-Marc Lasgouttes
+ *
+ * Full author contact details are available in file CREDITS.
+ */
+
+#ifndef LATEXPACKAGES_H
+#define LATEXPACKAGES_H
+
+#include <string>
+#include <set>
+
+
+namespace lyx {
+
+
+/** The list of avilable LaTeX packages
+ */
+class LaTeXPackages {
+public:
+	/// Which of the required packages are installed?
+	static void getAvailable();
+	/// Is the (required) package available?
+	static bool isAvailable(std::string const & name);
+private:
+	/// The available (required) packages
+	typedef std::set<std::string> Packages;
+	///
+	static Packages packages_;
+};
+
+
+} // namespace lyx
+
+#endif

src/Makefile.am

@@ -141,6 +141,7 @@ SOURCEFILESCORE = \
 	Language.cpp \
 	LaTeX.cpp \
 	LaTeXFeatures.cpp \
+	LaTeXPackages.cpp \
 	LayoutFile.cpp \
 	LayoutModuleList.cpp \
 	Length.cpp \
@@ -239,6 +240,7 @@ HEADERFILESCORE = \
 	KeySequence.h \
 	Language.h \
 	LaTeXFeatures.h \
+	LaTeXPackages.h \
 	LaTeX.h \
 	Layout.h \
 	LayoutEnums.h \

src/SpellChecker.h

@@ -14,7 +14,6 @@
 #define SPELL_BASE_H
 
 #include "support/strfwd.h"
-#include "support/lyxtime.h"
 
 
 namespace lyx {

src/frontends/qt4/GuiApplication.cpp

@@ -41,7 +41,7 @@
 #include "Intl.h"
 #include "KeyMap.h"
 #include "Language.h"
-#include "LaTeXFeatures.h"
+#include "LaTeXPackages.h"
 #include "Lexer.h"
 #include "LyX.h"
 #include "LyXAction.h"
@@ -1211,7 +1211,7 @@ void GuiApplication::reconfigure(string const & option)
 		current_view_->message(_("Reloading configuration..."));
 	lyxrc.read(libFileSearch(QString(), "lyxrc.defaults"), false);
 	// Re-read packages.lst
-	LaTeXFeatures::getAvailable();
+	LaTeXPackages::getAvailable();
 
 	if (ret)
 		Alert::information(_("System reconfiguration failed"),

src/frontends/qt4/GuiChanges.cpp

@@ -34,6 +34,7 @@ namespace lyx {
 namespace frontend {
 
 using support::bformat;
+using support::formatted_time;
 
 GuiChanges::GuiChanges(GuiView & lv)
 	: GuiDialog(lv, "changes", qt_("Merge Changes"))

src/support/lyxtime.cpp

@@ -12,9 +12,18 @@
 
 #include "support/lyxtime.h"
 
+#include "support/debug.h"
+#include "support/environment.h"
+#include "support/lstrings.h"
+#include "support/qstring_helpers.h"
+
+#include <QDateTime>
+#include <QLocale>
+
 using namespace std;
 
 namespace lyx {
+namespace support {
 
 time_t current_time()
 {
@@ -30,4 +39,63 @@ string const formatted_time(time_t t, string const & fmt)
 	return string(date);
 }
 
+
+time_t from_ctime(string t)
+{
+	// Example for the format: "Sun Nov  6 10:39:39 2011\n"
+	// Generously remove trailing '\n' (and other whitespace if needed)
+	t = trim(t, " \t\r\n");
+#if QT_VERSION >= 0x040400
+	// toDateTime() is too stupid to recognize variable amounts of
+	// whitespace (needed because ctime() outputs double spaces before
+	// single digit day numbers and hours)
+	t = subst(t, "  ", " ");
+	QString const format("ddd MMM d H:mm:ss yyyy");
+	QLocale loc("C");
+	QDateTime loc_dt = loc.toDateTime(toqstr(t), format);
+	if (!loc_dt.isValid()) {
+		LYXERR(Debug::LOCALE, "Could not parse `" << t
+				<< "´ (invalid format)");
+		return static_cast<time_t>(-1);
+	}
+	return loc_dt.toTime_t();
+#elif defined(_WIN32)
+#error "The minimum required Qt version on windows is Qt 4.4."
+#else
+	// strptime() is not available on windows (defined by POSIX)
+
+	// strptime() uses the current locale, so we need to switch to "C"
+	LYXERR(Debug::LOCALE, "Setting LC_ALL and LC_TIME to C");
+	string oldLC_ALL = getEnv("LC_ALL");
+	string oldLC_TIME = getEnv("LC_TIME");
+	if (!setEnv("LC_ALL", "C"))
+		LYXERR(Debug::LOCALE, "\t... LC_ALL failed!");
+	if (!setEnv("LC_TIME", "C"))
+		LYXERR(Debug::LOCALE, "\t... LC_TIME failed!");
+
+	struct tm loc_tm;
+	char const * const format = "%a%n%b%n%d%n%T%n%Y";
+	char * remainder = strptime(t.c_str(), format, &loc_tm);
+
+	LYXERR(Debug::LOCALE, "Resetting LC_ALL and LC_TIME");
+	if(!setEnv("LC_TIME", oldLC_TIME))
+		LYXERR(Debug::LOCALE, "\t... LC_TIME failed!");
+	if (!setEnv("LC_ALL", oldLC_ALL))
+		LYXERR(Debug::LOCALE, "\t... LC_ALL failed!");
+
+	if (!remainder) {
+		LYXERR(Debug::LOCALE, "Could not parse `" << t
+				<< "´ (invalid format)");
+		return static_cast<time_t>(-1);
+	}
+	if (*remainder != '\0') {
+		LYXERR(Debug::LOCALE, "Could not parse `" << t
+				<< "´ (excess characters)");
+		return static_cast<time_t>(-1);
+	}
+	return mktime(&loc_tm);
+#endif
+}
+
+} // namespace support
 } // namespace lyx

src/support/lyxtime.h

@@ -18,6 +18,7 @@
 
 
 namespace lyx {
+namespace support {
 
 time_t current_time();
 
@@ -27,6 +28,15 @@ time_t current_time();
  */
 std::string const formatted_time(time_t t, std::string const & fmt);
 
+/**
+ * Inverse of ctime().
+ * Since ctime() outputs the local time, the caller needs to ensure that the
+ * time zone and daylight saving time are the same as when \p t was created
+ * by ctime().
+ */
+time_t from_ctime(std::string t);
+
+} // namespace support
 } // namespace lyx
 
 #endif // LYXTIME_H

src/tex2lyx/Makefile.am

@@ -29,6 +29,7 @@ TEST_FILES = \
 	test/test-structure.tex
 
 LINKED_FILES = \
+	../Author.cpp \
 	../Color.cpp \
 	../Counters.cpp \
 	../Encoding.cpp \
@@ -36,6 +37,7 @@ LINKED_FILES = \
 	../Floating.cpp \
 	../FontInfo.cpp \
 	../insets/InsetLayout.cpp \
+	../LaTeXPackages.cpp \
 	../Layout.cpp \
 	../LayoutFile.cpp \
 	../LayoutModuleList.cpp \
@@ -57,7 +59,8 @@ tex2lyx_SOURCES = \
 	math.cpp \
 	Parser.cpp \
 	Parser.h \
-	preamble.cpp \
+	Preamble.cpp \
+	Preamble.h \
 	table.cpp \
 	tex2lyx.cpp \
 	tex2lyx.h \

src/tex2lyx/Parser.cpp

@@ -220,6 +220,20 @@ Token const Parser::next_token()
 }
 
 
+// We return a copy here because the tokens_ vector may get reallocated
+Token const Parser::next_next_token()
+{
+	static const Token dummy;
+	// If good() has not been called after the last get_token() we need
+	// to tokenize two more tokens.
+	if (pos_ + 1 >= tokens_.size()) {
+		tokenize_one();
+		tokenize_one();
+	}
+	return pos_ + 1 < tokens_.size() ? tokens_[pos_ + 1] : dummy;
+}
+
+
 // We return a copy here because the tokens_ vector may get reallocated
 Token const Parser::get_token()
 {
@@ -238,8 +252,7 @@ bool Parser::isParagraph()
 	if (curr_token().cat() == catNewline &&
 	    (curr_token().cs().size() > 1 ||
 	     (next_token().cat() == catSpace &&
-	      pos_ < tokens_.size() - 1 &&
+	      next_next_token().cat() == catNewline)))
-	      tokens_[pos_ + 1].cat() == catNewline)))
 		return true;
 	if (curr_token().cat() == catEscape && curr_token().cs() == "par")
 		return true;

src/tex2lyx/Parser.h

@@ -213,6 +213,8 @@ public:
 	Token const curr_token() const;
 	/// The next token.
 	Token const next_token();
+	/// The next but one token.
+	Token const next_next_token();
 	/// Make the next token current and return that.
 	Token const get_token();
 	/// \return whether the current token starts a new paragraph

src/tex2lyx/Preamble.cpp

@@ -1,5 +1,5 @@
 /**
- * \file preamble.cpp
+ * \file Preamble.cpp
  * This file is part of LyX, the document processor.
  * Licence details can be found in the file COPYING.
  *
@@ -13,6 +13,7 @@
 
 #include <config.h>
 
+#include "Preamble.h"
 #include "tex2lyx.h"
 
 #include "LayoutFile.h"
@@ -29,10 +30,6 @@
 
 #include <algorithm>
 #include <iostream>
-#include <sstream>
-#include <string>
-#include <vector>
-#include <map>
 
 using namespace std;
 using namespace lyx::support;
@@ -43,13 +40,7 @@ namespace lyx {
 // special columntypes
 extern map<char, int> special_columns;
 
-map<string, vector<string> > used_packages;
+Preamble preamble;
-const char * const modules_placeholder = "\001modules\001";
-
-// needed to handle encodings with babel
-bool one_language = true;
-string h_inputencoding = "auto";
-string h_paragraph_separation = "indent";
 
 namespace {
 
@@ -174,72 +165,16 @@ const char * const known_basic_color_codes[] = {"#0000ff", "#000000", "#00ffff",
 const char * const known_if_3arg_commands[] = {"@ifundefined", "IfFileExists",
 0};
 
-// default settings
+/// packages that work only in xetex
-ostringstream h_preamble;
+const char * const known_xetex_packages[] = {"arabxetex", "fixlatvian",
-string h_textclass               = "article";
+"fontbook", "fontwrap", "mathspec", "philokalia", "polyglossia", "unisugar",
-string h_use_default_options     = "false";
+"xeCJK", "xecolor", "xecyr", "xeindex", "xepersian", "xunicode", 0};
-string h_options;
+
-string h_language                = "english";
+// codes used to remove packages that are loaded automatically by LyX.
-string h_language_package        = "none";
+// Syntax: package_beg_sep<name>package_mid_sep<package loading code>package_end_sep
-string h_fontencoding            = "default";
+const char package_beg_sep = '\001';
-string h_font_roman              = "default";
+const char package_mid_sep = '\002';
-string h_font_sans               = "default";
+const char package_end_sep = '\003';
-string h_font_typewriter         = "default";
-string h_font_default_family     = "default";
-string h_font_sc                 = "false";
-string h_font_osf                = "false";
-string h_font_sf_scale           = "100";
-string h_font_tt_scale           = "100";
-string h_graphics                = "default";
-string h_float_placement;
-string h_paperfontsize           = "default";
-string h_spacing                 = "single";
-string h_use_hyperref            = "0";
-string h_pdf_title;
-string h_pdf_author;
-string h_pdf_subject;
-string h_pdf_keywords;
-string h_pdf_bookmarks           = "1";
-string h_pdf_bookmarksnumbered   = "0";
-string h_pdf_bookmarksopen       = "0";
-string h_pdf_bookmarksopenlevel  = "1";
-string h_pdf_breaklinks          = "0";
-string h_pdf_pdfborder           = "0";
-string h_pdf_colorlinks          = "0";
-string h_pdf_backref             = "section";
-string h_pdf_pdfusetitle         = "1";
-string h_pdf_pagemode;
-string h_pdf_quoted_options;
-string h_papersize               = "default";
-string h_use_geometry            = "false";
-string h_use_amsmath             = "1";
-string h_use_esint               = "1";
-string h_use_mhchem              = "0";
-string h_use_mathdots            = "0";
-string h_cite_engine             = "basic";
-string h_use_bibtopic            = "false";
-string h_paperorientation        = "portrait";
-string h_suppress_date           = "false";
-string h_use_refstyle            = "0";
-string h_backgroundcolor;
-string h_boxbgcolor;
-string h_fontcolor;
-string h_notefontcolor;
-string h_secnumdepth             = "3";
-string h_tocdepth                = "3";
-string h_defskip                 = "medskip";
-string h_paragraph_indentation   = "default";
-string h_quotes_language         = "english";
-string h_papercolumns            = "1";
-string h_papersides;
-string h_paperpagestyle          = "default";
-string h_listings_params;
-string h_tracking_changes        = "false";
-string h_output_changes          = "false";
-string h_html_math_output        = "0";
-string h_html_css_as_file        = "0";
-string h_html_be_strict          = "false";
-string h_margins;
 
 
 // returns true if at least one of the options in what has been found
@@ -338,12 +273,74 @@ string process_keyval_opt(vector<string> & options, string name)
 	return "";
 }
 
+} // anonymous namespace
 
-/*!
+
- * Add package \p name with options \p options to used_packages.
+bool Preamble::indentParagraphs() const
- * Remove options from \p options that we don't want to output.
+{
- */
+	return h_paragraph_separation == "indent";
-void add_package(string const & name, vector<string> & options)
+}
+
+
+bool Preamble::isPackageUsed(string const & package) const
+{
+	return used_packages.find(package) != used_packages.end();
+}
+
+
+vector<string> Preamble::getPackageOptions(string const & package) const
+{
+	map<string, vector<string> >::const_iterator it = used_packages.find(package);
+	if (it != used_packages.end())
+		return it->second;
+	return vector<string>();
+}
+
+
+void Preamble::registerAutomaticallyLoadedPackage(std::string const & package)
+{
+	auto_packages.insert(package);
+}
+
+
+void Preamble::addModule(string const & module)
+{
+	used_modules.push_back(module);
+}
+
+
+void Preamble::suppressDate(bool suppress)
+{
+	if (suppress)
+		h_suppress_date = "true";
+	else
+		h_suppress_date = "false";
+}
+
+
+void Preamble::registerAuthor(std::string const & name)
+{
+	Author author(from_utf8(name), empty_docstring());
+	author.setUsed(true);
+	authors_.record(author);
+	h_tracking_changes = "true";
+	h_output_changes = "true";
+}
+
+
+Author const & Preamble::getAuthor(std::string const & name) const
+{
+	Author author(from_utf8(name), empty_docstring());
+	for (AuthorList::Authors::const_iterator it = authors_.begin();
+	     it != authors_.end(); it++)
+		if (*it == author)
+			return *it;
+	static Author const dummy;
+	return dummy;
+}
+
+
+void Preamble::add_package(string const & name, vector<string> & options)
 {
 	// every package inherits the global options
 	if (used_packages.find(name) == used_packages.end())
@@ -362,6 +359,8 @@ void add_package(string const & name, vector<string> & options)
 }
 
 
+namespace {
+
 // Given is a string like "scaled=0.9", return 0.9 * 100
 string const scale_as_percentage(string const & scale)
 {
@@ -386,8 +385,82 @@ string remove_braces(string const & value)
 	return value;
 }
 
+} // anonymous namespace
 
-void handle_hyperref(vector<string> & options)
+
+Preamble::Preamble() : one_language(true)
+{
+	//h_backgroundcolor;
+	//h_boxbgcolor;
+	h_cite_engine             = "basic";
+	h_defskip                 = "medskip";
+	//h_float_placement;
+	//h_fontcolor;
+	h_fontencoding            = "default";
+	h_font_roman              = "default";
+	h_font_sans               = "default";
+	h_font_typewriter         = "default";
+	h_font_default_family     = "default";
+	h_font_sc                 = "false";
+	h_font_osf                = "false";
+	h_font_sf_scale           = "100";
+	h_font_tt_scale           = "100";
+	h_graphics                = "default";
+	h_html_be_strict          = "false";
+	h_html_css_as_file        = "0";
+	h_html_math_output        = "0";
+	h_inputencoding           = "auto";
+	h_language                = "english";
+	h_language_package        = "none";
+	//h_listings_params;
+	//h_margins;
+	//h_notefontcolor;
+	//h_options;
+	h_output_changes          = "false";
+	h_papercolumns            = "1";
+	h_paperfontsize           = "default";
+	h_paperorientation        = "portrait";
+	h_paperpagestyle          = "default";
+	//h_papersides;
+	h_papersize               = "default";
+	h_paragraph_indentation   = "default";
+	h_paragraph_separation    = "indent";
+	//h_pdf_title;
+	//h_pdf_author;
+	//h_pdf_subject;
+	//h_pdf_keywords;
+	h_pdf_bookmarks           = "1";
+	h_pdf_bookmarksnumbered   = "0";
+	h_pdf_bookmarksopen       = "0";
+	h_pdf_bookmarksopenlevel  = "1";
+	h_pdf_breaklinks          = "0";
+	h_pdf_pdfborder           = "0";
+	h_pdf_colorlinks          = "0";
+	h_pdf_backref             = "section";
+	h_pdf_pdfusetitle         = "1";
+	//h_pdf_pagemode;
+	//h_pdf_quoted_options;
+	h_quotes_language         = "english";
+	h_secnumdepth             = "3";
+	h_spacing                 = "single";
+	h_suppress_date           = "false";
+	h_textclass               = "article";
+	h_tocdepth                = "3";
+	h_tracking_changes        = "false";
+	h_use_bibtopic            = "false";
+	h_use_indices             = "false";
+	h_use_geometry            = "false";
+	h_use_amsmath             = "1";
+	h_use_default_options     = "false";
+	h_use_esint               = "1";
+	h_use_hyperref            = "0";
+	h_use_mhchem              = "0";
+	h_use_mathdots            = "0";
+	h_use_refstyle            = "0";
+}
+
+
+void Preamble::handle_hyperref(vector<string> & options)
 {
 	// FIXME swallow inputencoding changes that might surround the
 	//       hyperref setup if it was written by LyX
@@ -471,13 +544,46 @@ void handle_hyperref(vector<string> & options)
 }
 
 
-void handle_package(Parser &p, string const & name, string const & opts,
+void Preamble::handle_geometry(vector<string> & options)
-		    bool in_lyx_preamble)
+{
+	h_use_geometry = "true";
+	vector<string>::iterator it;
+	// paper orientation
+	if ((it = find(options.begin(), options.end(), "landscape")) != options.end()) {
+		h_paperorientation = "landscape";
+		options.erase(it);
+	}
+	// paper size
+	// keyval version: "paper=letter"
+	string paper = process_keyval_opt(options, "paper");
+	if (!paper.empty())
+		h_papersize = paper + "paper";
+	// alternative version: "letterpaper"
+	handle_opt(options, known_paper_sizes, h_papersize);
+	delete_opt(options, known_paper_sizes);
+	// page margins
+	char const * const * margin = known_paper_margins;
+	for (; *margin; ++margin) {
+		string value = process_keyval_opt(options, *margin);
+		if (!value.empty()) {
+			int k = margin - known_paper_margins;
+			string name = known_coded_paper_margins[k];
+			h_margins += '\\' + name + ' ' + value + '\n';
+		}
+	}
+}
+
+
+void Preamble::handle_package(Parser &p, string const & name,
+                              string const & opts, bool in_lyx_preamble)
 {
 	vector<string> options = split_options(opts);
 	add_package(name, options);
 	string scale;
 
+	if (is_known(name, known_xetex_packages))
+		xetex = true;
+
 	// roman fonts
 	if (is_known(name, known_roman_fonts)) {
 		h_font_roman = name;
@@ -596,64 +702,53 @@ void handle_package(Parser &p, string const & name, string const & opts,
 	else if (is_known(name, known_old_language_packages)) {
 		// known language packages from the times before babel
 		// if they are found and not also babel, they will be used as
-		// cutom language package
+		// custom language package
 		h_language_package = "\\usepackage{" + name + "}";
 	}
 
-	else if (name == "makeidx")
-		; // ignore this
-
 	else if (name == "prettyref")
-		; // ignore this
+		; // ignore this FIXME: Use the package separator mechanism instead
 
 	else if (name == "varioref")
-		; // ignore this
+		; // ignore this FIXME: Use the package separator mechanism instead
 
 	else if (name == "verbatim")
-		; // ignore this
+		; // ignore this FIXME: Use the package separator mechanism instead
-
-	else if (name == "nomencl")
-		; // ignore this
 
 	else if (name == "textcomp")
-		; // ignore this
+		; // ignore this FIXME: Use the package separator mechanism instead
 
-	else if (name == "url")
+	else if (name == "lyxskak") {
-		; // ignore this
+		// ignore this and its options
+		if (!options.empty())
+			options.clear();
+	}
 
-	else if (name == "subscript")
+	else if (name == "array" || name == "booktabs" || name == "float" ||
-		; // ignore this
+	         name == "color" || name == "hhline" || name == "longtable" ||
-
+	         name == "makeidx" || name == "nomencl" || name == "splitidx" ||
-	else if (name == "color") {
+	         name == "setspace" || name == "subscript" || name == "ulem" ||
-		// with the following command this package is only loaded when needed for
+	         name == "url") {
-		// undefined colors, since we only support the predefined colors
+		if (!in_lyx_preamble)
-		h_preamble << "\\@ifundefined{definecolor}\n {\\usepackage{color}}{}\n";
+			h_preamble << package_beg_sep << name
+			           << package_mid_sep << "\\usepackage{"
+			           << name << '}' << package_end_sep;
 	}
 
 	else if (name == "graphicx")
-		; // ignore this
+		; // ignore this FIXME: Use the package separator mechanism instead
-
-	else if (name == "setspace")
-		; // ignore this
-
-#if 0
-	// do not ignore as long as we don't support all commands (e.g. \xout is missing)
-	else if (name == "ulem")
-		; // ignore this
-#endif
 
 	else if (name == "geometry")
-		; // Ignore this, the geometry settings are made by the \geometry
+		handle_geometry(options);
-		  // command. This command is handled below.
 
 	else if (name == "rotfloat")
-		; // ignore this
+		; // ignore this FIXME: Use the package separator mechanism instead
 
 	else if (name == "wrapfig")
-		; // ignore this
+		; // ignore this FIXME: Use the package separator mechanism instead
 
 	else if (name == "subfig")
-		; // ignore this
+		; // ignore this FIXME: Use the package separator mechanism instead
 
 	else if (is_known(name, known_languages))
 		h_language = name;
@@ -699,7 +794,7 @@ void handle_package(Parser &p, string const & name, string const & opts,
 }
 
 
-void handle_if(Parser & p, bool in_lyx_preamble)
+void Preamble::handle_if(Parser & p, bool in_lyx_preamble)
 {
 	while (p.good()) {
 		Token t = p.get_token();
@@ -716,7 +811,7 @@ void handle_if(Parser & p, bool in_lyx_preamble)
 }
 
 
-void end_preamble(ostream & os, TextClass const & /*textclass*/)
+bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
 {
 	// translate from babel to LyX names
 	h_language = babel2lyx(h_language);
@@ -748,19 +843,53 @@ void end_preamble(ostream & os, TextClass const & /*textclass*/)
 	else if (is_known(h_language, known_english_quotes_languages))
 		h_quotes_language = "english";
 
+	if (contains(h_float_placement, "H"))
+		registerAutomaticallyLoadedPackage("float");
+	if (h_spacing != "single" && h_spacing != "default")
+		registerAutomaticallyLoadedPackage("setspace");
+
 	// output the LyX file settings
 	os << "#LyX file created by tex2lyx " << PACKAGE_VERSION << "\n"
 	   << "\\lyxformat " << LYX_FORMAT << '\n'
 	   << "\\begin_document\n"
 	   << "\\begin_header\n"
 	   << "\\textclass " << h_textclass << "\n";
-	if (!h_preamble.str().empty())
+	string const raw = subdoc ? empty_string() : h_preamble.str();
-		os << "\\begin_preamble\n" << h_preamble.str() << "\n\\end_preamble\n";
+	if (!raw.empty()) {
+		os << "\\begin_preamble\n";
+		for (string::size_type i = 0; i < raw.size(); ++i) {
+			if (raw[i] == package_beg_sep) {
+				// Here follows some package loading code that
+				// must be skipped if the package is loaded
+				// automatically.
+				string::size_type j = raw.find(package_mid_sep, i);
+				if (j == string::npos)
+					return false;
+				string::size_type k = raw.find(package_end_sep, j);
+				if (k == string::npos)
+					return false;
+				string const package = raw.substr(i + 1, j - i - 1);
+				string const replacement = raw.substr(j + 1, k - j - 1);
+				if (auto_packages.find(package) == auto_packages.end())
+					os << replacement;
+				i = k;
+			} else
+				os.put(raw[i]);
+		}
+		os << "\n\\end_preamble\n";
+	}
 	if (!h_options.empty())
 		os << "\\options " << h_options << "\n";
-	os << "\\use_default_options " << h_use_default_options << "\n"
+	os << "\\use_default_options " << h_use_default_options << "\n";
-	   << modules_placeholder
+	if (!used_modules.empty()) {
-	   << "\\language " << h_language << "\n"
+		os << "\\begin_modules\n";
+		vector<string>::const_iterator const end = used_modules.end();
+		vector<string>::const_iterator it = used_modules.begin();
+		for (; it != end; it++)
+			os << *it << '\n';
+		os << "\\end_modules\n";
+	}
+	os << "\\language " << h_language << "\n"
 	   << "\\language_package " << h_language_package << "\n"
 	   << "\\inputencoding " << h_inputencoding << "\n"
 	   << "\\fontencoding " << h_fontencoding << "\n"
@@ -809,6 +938,7 @@ void end_preamble(ostream & os, TextClass const & /*textclass*/)
 	   << "\\use_mathdots " << h_use_mathdots << "\n"
 	   << "\\cite_engine " << h_cite_engine << "\n"
 	   << "\\use_bibtopic " << h_use_bibtopic << "\n"
+	   << "\\use_indices " << h_use_indices << "\n"
 	   << "\\paperorientation " << h_paperorientation << '\n'
 	   << "\\suppress_date " << h_suppress_date << '\n'
 	   << "\\use_refstyle " << h_use_refstyle << '\n';
@@ -839,17 +969,15 @@ void end_preamble(ostream & os, TextClass const & /*textclass*/)
 	   << "\\html_math_output " << h_html_math_output << "\n"
 	   << "\\html_css_as_file " << h_html_css_as_file << "\n"
 	   << "\\html_be_strict " << h_html_be_strict << "\n"
+	   << authors_
 	   << "\\end_header\n\n"
 	   << "\\begin_body\n";
-	// clear preamble for subdocuments
+	return true;
-	h_preamble.str("");
 }
 
-} // anonymous namespace
 
-
+void Preamble::parse(Parser & p, string const & forceclass,
-void parse_preamble(Parser & p, ostream & os, 
+                     TeX2LyXDocClass & tc)
-	string const & forceclass, TeX2LyXDocClass & tc)
 {
 	// initialize fixed types
 	special_columns['D'] = 3;
@@ -933,15 +1061,22 @@ void parse_preamble(Parser & p, ostream & os,
 		}
 
 		else if (t.cs() == "color") {
+			string const space =
+				(p.hasOpt() ? p.getOpt() : string());
 			string argument = p.getArg('{', '}');
 			// check the case that a standard color is used
-			if (is_known(argument, known_basic_colors))
+			if (space.empty() && is_known(argument, known_basic_colors)) {
-				h_fontcolor = color2code(argument);
+				h_fontcolor = rgbcolor2code(argument);
+				preamble.registerAutomaticallyLoadedPackage("color");
+			} else if (space.empty() && argument == "document_fontcolor")
+				preamble.registerAutomaticallyLoadedPackage("color");
 			// check the case that LyX's document_fontcolor is defined
 			// but not used for \color
-			if (argument != "document_fontcolor"
+			else {
-				&& !is_known(argument, known_basic_colors)) {
+				h_preamble << t.asInput();
-				h_preamble << t.asInput() << '{' << argument << '}';
+				if (!space.empty())
+					h_preamble << space;
+				h_preamble << '{' << argument << '}';
 				// the color might already be set because \definecolor
 				// is parsed before this
 				h_fontcolor = "";
@@ -951,12 +1086,13 @@ void parse_preamble(Parser & p, ostream & os,
 		else if (t.cs() == "pagecolor") {
 			string argument = p.getArg('{', '}');
 			// check the case that a standard color is used
-			if (is_known(argument, known_basic_colors))
+			if (is_known(argument, known_basic_colors)) {
-				h_backgroundcolor = color2code(argument);
+				h_backgroundcolor = rgbcolor2code(argument);
+			} else if (argument == "page_backgroundcolor")
+				preamble.registerAutomaticallyLoadedPackage("color");
 			// check the case that LyX's page_backgroundcolor is defined
 			// but not used for \pagecolor
-			if (argument != "page_backgroundcolor"
+			else {
-				&& !is_known(argument, known_basic_colors)) {
 				h_preamble << t.asInput() << '{' << argument << '}';
 				// the color might already be set because \definecolor
 				// is parsed before this
@@ -1006,6 +1142,11 @@ void parse_preamble(Parser & p, ostream & os,
 				h_font_default_family = family.erase(0,1);
 			}
 
+			// remove the lyxdot definition that is re-added by LyX
+			// if necessary
+			if (name == "\\lyxdot")
+				in_lyx_preamble = true;
+
 			// Add the command to the known commands
 			add_known_command(name, opt1, !opt2.empty(), from_utf8(body));
 
@@ -1069,7 +1210,7 @@ void parse_preamble(Parser & p, ostream & os,
 				opts.erase(it);
 			}
 			// paper sizes
-			// some size options are know to any document classes, other sizes
+			// some size options are known to any document classes, other sizes
 			// are handled by the \geometry command of the geometry package
 			handle_opt(opts, known_class_paper_sizes, h_papersize);
 			delete_opt(opts, known_class_paper_sizes);
@@ -1189,32 +1330,8 @@ void parse_preamble(Parser & p, ostream & os,
 		}
 
 		else if (t.cs() == "geometry") {
-			h_use_geometry = "true";
 			vector<string> opts = split_options(p.getArg('{', '}'));
-			vector<string>::iterator it;
+			handle_geometry(opts);
-			// paper orientation
-			if ((it = find(opts.begin(), opts.end(), "landscape")) != opts.end()) {
-				h_paperorientation = "landscape";
-				opts.erase(it);
-			}
-			// paper size
-			handle_opt(opts, known_paper_sizes, h_papersize);
-			delete_opt(opts, known_paper_sizes);
-			// page margins
-			char const * const * margin = known_paper_margins;
-			int k = -1;
-			for (; *margin; ++margin) {
-				k += 1;
-				// search for the "=" in e.g. "lmargin=2cm" to get the value
-				for(size_t i = 0; i != opts.size(); i++) {
-					if (opts.at(i).find(*margin) != string::npos) {
-						string::size_type pos = opts.at(i).find("=");
-						string value = opts.at(i).substr(pos + 1);
-						string name = known_coded_paper_margins[k];
-						h_margins += "\\" + name + " " + value + "\n";
-					}
-				}
-			}
 		}
 
 		else if (t.cs() == "definecolor") {
@@ -1273,13 +1390,27 @@ void parse_preamble(Parser & p, ostream & os,
 			string const arg2 = p.verbatim_item();
 			string const arg3 = p.verbatim_item();
 			// test case \@ifundefined{date}{}{\date{}}
-			if (arg1 == "date" && arg2.empty() && arg3 == "\\date{}") {
+			if (t.cs() == "@ifundefined" && arg1 == "date" &&
+			    arg2.empty() && arg3 == "\\date{}") {
 				h_suppress_date = "true";
+			// older tex2lyx versions did output
+			// \@ifundefined{definecolor}{\usepackage{color}}{}
+			} else if (t.cs() == "@ifundefined" &&
+			           arg1 == "definecolor" &&
+			           arg2 == "\\usepackage{color}" &&
+			           arg3.empty()) {
+				if (!in_lyx_preamble)
+					h_preamble << package_beg_sep
+					           << "color"
+					           << package_mid_sep
+					           << "\\@ifundefined{definecolor}{color}{}"
+					           << package_end_sep;
 			// test for case
 			//\@ifundefined{showcaptionsetup}{}{%
 			// \PassOptionsToPackage{caption=false}{subfig}}
 			// that LyX uses for subfloats
-			} else if (arg1 == "showcaptionsetup" && arg2.empty()
+			} else if (t.cs() == "@ifundefined" &&
+			           arg1 == "showcaptionsetup" && arg2.empty()
 				&& arg3 == "%\n \\PassOptionsToPackage{caption=false}{subfig}") {
 				; // do nothing
 			} else if (!in_lyx_preamble) {
@@ -1320,11 +1451,9 @@ void parse_preamble(Parser & p, ostream & os,
 		ss << tc.sides();
 		h_papersides = ss.str();
 	}
-	end_preamble(os, tc);
 }
 
 
-/// translates a babel language name to a LyX language name
 string babel2lyx(string const & language)
 {
 	char const * const * where = is_known(language, known_languages);
@@ -1334,13 +1463,16 @@ string babel2lyx(string const & language)
 }
 
 
-/// translates a color name to a LyX color code
+string rgbcolor2code(string const & name)
-string color2code(string const & name)
 {
 	char const * const * where = is_known(name, known_basic_colors);
-	if (where)
+	if (where) {
+		// "red", "green" etc
 		return known_basic_color_codes[where - known_basic_colors];
-	return name;
+	}
+	// "255,0,0", "0,255,0" etc
+	RGBColor c(RGBColorFromLaTeX(name));
+	return X11hexname(c);
 }
 
 // }])

src/tex2lyx/Preamble.h

@@ -0,0 +1,178 @@
+/**
+ * \file Preamble.h
+ * This file is part of LyX, the document processor.
+ * Licence details can be found in the file COPYING.
+ *
+ * \author André Pönitz
+ * \author Uwe Stöhr
+ *
+ * Full author contact details are available in file CREDITS.
+ */
+
+// {[(
+
+#ifndef LYX_PREAMBLE_H
+#define LYX_PREAMBLE_H
+
+#include "Author.h"
+
+#include <iosfwd>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <map>
+#include <set>
+
+
+namespace lyx {
+
+class Parser;
+class TeX2LyXDocClass;
+
+class Preamble
+{
+public:
+	Preamble();
+
+	///
+	std::string inputencoding() const { return h_inputencoding; }
+	///
+	std::string notefontcolor() const { return h_notefontcolor; }
+	///
+	std::string use_indices() const { return h_use_indices; }
+	///
+	bool indentParagraphs() const;
+	///
+	bool isPackageUsed(std::string const & package) const;
+	///
+	std::vector<std::string>
+	getPackageOptions(std::string const & package) const;
+	/// Tell that \p package will be loaded automatically by LyX.
+	/// This has only an effect if \p package is prepared for
+	/// autoloading in parse().
+	void registerAutomaticallyLoadedPackage(std::string const & package);
+	///
+	void addModule(std::string const & module);
+	///
+	void suppressDate(bool suppress);
+	/// Register an author named \p name in the author list
+	void registerAuthor(std::string const & name);
+	/// Get author named \p name (must be registered first)
+	Author const & getAuthor(std::string const & name) const;
+
+
+	/// Parses the LaTeX preamble into internal data
+	void parse(Parser & p, std::string const & forceclass,
+	           TeX2LyXDocClass & tc);
+	/// Writes the LyX file header from internal data
+	bool writeLyXHeader(std::ostream & os, bool subdoc);
+
+private:
+	///
+	std::map<std::string, std::vector<std::string> > used_packages;
+	/// Packages that will be loaded automatically by LyX
+	std::set<std::string> auto_packages;
+	///
+	std::vector<std::string> used_modules;
+
+	/// needed to handle encodings with babel
+	bool one_language;
+
+	std::ostringstream h_preamble;
+	std::string h_backgroundcolor;
+	std::string h_boxbgcolor;
+	std::string h_cite_engine;
+	std::string h_defskip;
+	std::string h_float_placement;
+	std::string h_fontcolor;
+	std::string h_fontencoding;
+	std::string h_font_roman;
+	std::string h_font_sans;
+	std::string h_font_typewriter;
+	std::string h_font_default_family;
+	std::string h_font_sc;
+	std::string h_font_osf;
+	std::string h_font_sf_scale;
+	std::string h_font_tt_scale;
+	std::string h_graphics;
+	std::string h_html_be_strict;
+	std::string h_html_css_as_file;
+	std::string h_html_math_output;
+	std::string h_inputencoding;
+	std::string h_language;
+	std::string h_language_package;
+	std::string h_listings_params;
+	std::string h_margins;
+	std::string h_notefontcolor;
+	std::string h_options;
+	std::string h_output_changes;
+	std::string h_papercolumns;
+	std::string h_paperfontsize;
+	std::string h_paperorientation;
+	std::string h_paperpagestyle;
+	std::string h_papersides;
+	std::string h_papersize;
+	std::string h_paragraph_indentation;
+	/// necessary to set the separation when \setlength is parsed
+	std::string h_paragraph_separation;
+	std::string h_pdf_title;
+	std::string h_pdf_author;
+	std::string h_pdf_subject;
+	std::string h_pdf_keywords;
+	std::string h_pdf_bookmarks;
+	std::string h_pdf_bookmarksnumbered;
+	std::string h_pdf_bookmarksopen;
+	std::string h_pdf_bookmarksopenlevel;
+	std::string h_pdf_breaklinks;
+	std::string h_pdf_pdfborder;
+	std::string h_pdf_colorlinks;
+	std::string h_pdf_backref;
+	std::string h_pdf_pdfusetitle;
+	std::string h_pdf_pagemode;
+	std::string h_pdf_quoted_options;
+	std::string h_quotes_language;
+	std::string h_secnumdepth;
+	std::string h_spacing;
+	std::string h_suppress_date;
+	std::string h_textclass;
+	std::string h_tocdepth;
+	std::string h_tracking_changes;
+	std::string h_use_bibtopic;
+	std::string h_use_indices;
+	std::string h_use_geometry;
+	std::string h_use_amsmath;
+	std::string h_use_default_options;
+	std::string h_use_esint;
+	std::string h_use_hyperref;
+	std::string h_use_mhchem;
+	std::string h_use_mathdots;
+	std::string h_use_refstyle;
+
+	/*!
+	 * Add package \p name with options \p options to used_packages.
+	 * Remove options from \p options that we don't want to output.
+	 */
+	void add_package(std::string const & name,
+	                 std::vector<std::string> & options);
+	///
+	void handle_hyperref(std::vector<std::string> & options);
+	///
+	void handle_geometry(std::vector<std::string> & options);
+	///
+	void handle_package(Parser &p, std::string const & name,
+	                    std::string const & opts, bool in_lyx_preamble);
+	///
+	void handle_if(Parser & p, bool in_lyx_preamble);
+
+	AuthorList authors_;
+};
+
+
+extern Preamble preamble;
+
+// }])
+
+
+} // namespace lyx
+
+#endif

src/tex2lyx/TODO.txt

@@ -10,15 +10,16 @@ LyX feature:   LyX inset or document setting
 
 
 Format LaTeX feature                        LyX feature
-222    change tracking                      change tracking
 224    external insets defined in           InsetExternal
-       lib/external_templates. This is
+       lib/external_templates.
-       quite difficult to recognize.
+       (Date and RasterImage cannot be supported
+       (Chess diagram and Spreadsheet are supported)
+       (Xfig figure, Lilypond, Dia diagram can be supported by looking at the file extension)
+       (for PDFpages work is in progress by uwestoehr)
 226    nothing (impossible to import)       InsetBranch, \branch...\end_branch
 226    transformations                      InsetExternal
 228    draft                                InsetExternal
 232    bibtopic                             InsetBibTeX
-248    booktabs.sty                         InsetTabular
 254    esint.sty                            \use_esint
 266    armenian                             \language, \lang
 267    XeTeX                                utf8 encoding
@@ -45,11 +46,8 @@ Format LaTeX feature                        LyX feature
 363    horizontal longtable alignment       InsetTabular
 364    branch file name suffix              \filename_suffix
 366    relative lengths for parskip         \defskip
-367    relative lengths for h and v space   InsetHSpace, InsetVSpace
+367    relative lengths for h and v space   InsetSpace, InsetVSpace
-368    glue lengths                         InsetHSpace
+368    glue lengths                         InsetSpace
-369    author id                            \author
-370    \date{}                              \suppress_date
-                                            (partly supported, see bug #7844)
 371    automatic mhchem loading             \use_mhchem
 375    \includeonly                         \{begin,end}_includeonly
 376    update .aux of unincluded children   \maintain_unincluded_children
@@ -66,9 +64,7 @@ Format LaTeX feature                        LyX feature
 401    feyn.sty                             InsetMathDiagram
 402    \addcontentsline                     InsetBibtex bibtotoc option
 404    refstyle.sty                         InsetRef
-405    author hash                          \author
 407    vertical offset for multirows        InsetTabular
 409    XeTeX                                \use_non_tex_fonts
 411    support for polyglossia              \language_package	(the cases of no package, of babel and of custom package is supported)
-412    tabular*                             InsetTabular
 

src/tex2lyx/math.cpp

@@ -27,6 +27,16 @@ bool is_math_env(string const & name)
 }
 
 
+bool is_display_math_env(string const & name)
+{
+	CommandMap::const_iterator it = known_math_environments.find(name);
+	if (it != known_math_environments.end())
+		if (!it->second.empty())
+			return it->second.back() == displaymath;
+	return false;
+}
+
+
 void parse_math(Parser & p, ostream & os, unsigned flags, const mode_type mode)
 {
 	while (p.good()) {

src/tex2lyx/table.cpp

@@ -16,6 +16,8 @@
 
 #include "tex2lyx.h"
 
+#include "Preamble.h"
+
 #include "support/lassert.h"
 #include "support/convert.h"
 #include "support/lstrings.h"
@@ -73,10 +75,23 @@ class RowInfo {
 public:
 	RowInfo() : topline(false), bottomline(false), type(LT_NORMAL),
 		    caption(false), newpage(false) {}
+	/// Does this row have any special setting?
+	bool special() const
+	{
+		return topline || bottomline || !top_space.empty() ||
+			!bottom_space.empty() || !interline_space.empty() ||
+			type != LT_NORMAL || caption || newpage;
+	}
 	/// horizontal line above
 	bool topline;
 	/// horizontal line below
 	bool bottomline;
+	/// Extra space between the top line and this row
+	string top_space;
+	/// Extra space between this row and the bottom line
+	string bottom_space;
+	/// Extra space between the bottom line and the next top line
+	string interline_space;
 	/// These are for longtabulars only
 	/// row type (head, foot, firsthead etc.)
 	LTRowType type;
@@ -129,6 +144,23 @@ public:
 };
 
 
+class ltType {
+public:
+	// constructor
+	ltType() : topDL(false), bottomDL(false), empty(false) {}
+	// we have this header type (is set in the getLT... functions)
+	bool set;
+	// double borders on top
+	bool topDL;
+	// double borders on bottom
+	bool bottomDL;
+	// used for FirstHeader & LastFooter and if this is true
+	// all the rows marked as FirstHeader or LastFooter are
+	// ignored in the output and it is set to be empty!
+	bool empty;
+};
+
+
 /// translate a horizontal alignment (as stored in ColInfo and CellInfo) to LyX
 inline char const * verbose_align(char c)
 {
@@ -471,12 +503,35 @@ bool parse_hlines(Parser & p, Token const & t, string & hlines,
 {
 	LASSERT(t.cat() == catEscape, return false);
 
-	if (t.cs() == "hline")
+	if (t.cs() == "hline" || t.cs() == "toprule" || t.cs() == "midrule" ||
-		hlines += "\\hline";
+	    t.cs() == "bottomrule")
+		hlines += '\\' + t.cs();
 
 	else if (t.cs() == "cline")
 		hlines += "\\cline{" + p.verbatim_item() + '}';
 
+	else if (t.cs() == "cmidrule") {
+		// We cannot handle the \cmidrule(l){3-4} form
+		p.pushPosition();
+		p.skip_spaces(true);
+		bool const hasParentheses(p.getFullArg('(', ')').first);
+		p.popPosition();
+		if (hasParentheses)
+			return false;
+		hlines += "\\cmidrule{" + p.verbatim_item() + '}';
+	}
+
+	else if (t.cs() == "addlinespace") {
+		p.pushPosition();
+		p.skip_spaces(true);
+		bool const hasArgument(p.getFullArg('{', '}').first);
+		p.popPosition();
+		if (hasArgument)
+			hlines += "\\addlinespace{" + p.verbatim_item() + '}';
+		else
+			hlines += "\\addlinespace";
+	}
+
 	else if (is_long_tabular && t.cs() == "newpage")
 		hlines += "\\newpage";
 
@@ -610,7 +665,6 @@ void parse_table(Parser & p, ostream & os, bool is_long_tabular,
 				}
 				continue;
 			}
-
 		}
 
 		// We need a HLINE separator if we either have no hline
@@ -628,14 +682,20 @@ void parse_table(Parser & p, ostream & os, bool is_long_tabular,
 			pos = IN_COLUMNS;
 			break;
 		case IN_HLINES_END:
-			// Oops, there is still cell content after hline
+			// Oops, there is still cell content or unsupported
-			// stuff. This does not work in LaTeX, so we ignore
+			// booktabs commands after hline stuff. The latter are
-			// the hlines.
+			// moved to the cell, and the first does not work in
-			cerr << "Ignoring '" << hlines << "' in a cell"
+			// LaTeX, so we ignore the hlines.
-			     << endl;
 			os << comments;
-			hlines.erase();
 			comments.erase();
+			if (support::contains(hlines, "\\hline") ||
+			    support::contains(hlines, "\\cline") ||
+			    support::contains(hlines, "\\newpage"))
+				cerr << "Ignoring '" << hlines
+				     << "' in a cell" << endl;
+			else
+				os << hlines;
+			hlines.erase();
 			pos = IN_COLUMNS;
 			break;
 		case IN_COLUMNS:
@@ -770,9 +830,11 @@ void handle_hline_below(RowInfo & ri, vector<CellInfo> & ci)
 } // anonymous namespace
 
 
-void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
+void handle_tabular(Parser & p, ostream & os, string const & name,
-		    Context & context)
+                    string const & tabularwidth, Context & context)
 {
+	bool const is_long_tabular(name == "longtable");
+	bool booktabs = false;
 	string tabularvalignment("middle");
 	string posopts = p.getOpt();
 	if (!posopts.empty()) {
@@ -806,13 +868,18 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 
 	vector< vector<CellInfo> > cellinfo(lines.size());
 	vector<RowInfo> rowinfo(lines.size());
+	ltType endfirsthead;
+	ltType endhead;
+	ltType endfoot;
+	ltType endlastfoot;
 
 	// split into rows
 	//cerr << "// split into rows\n";
-	for (size_t row = 0; row < rowinfo.size(); ++row) {
+	for (size_t row = 0; row < rowinfo.size();) {
 
 		// init row
 		cellinfo[row].resize(colinfo.size());
+		bool deletelastrow = false;
 
 		// split row
 		vector<string> dummy;
@@ -840,13 +907,18 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 			while (p1.good()) {
 				Token t = p1.get_token();
 				//cerr << "read token: " << t << "\n";
-				if (t.cs() == "hline") {
+				if (t.cs() == "hline" || t.cs() == "toprule" ||
+				    t.cs() == "midrule" ||
+				    t.cs() == "bottomrule") {
+					if (t.cs() != "hline")
+						booktabs = true;
 					if (i == 0) {
 						if (rowinfo[row].topline) {
 							if (row > 0) // extra bottomline above
 								handle_hline_below(rowinfo[row - 1], cellinfo[row - 1]);
 							else
-								cerr << "dropping extra hline\n";
+								cerr << "dropping extra "
+								     << t.cs() << '\n';
 							//cerr << "below row: " << row-1 << endl;
 						} else {
 							handle_hline_above(rowinfo[row], cellinfo[row]);
@@ -856,37 +928,39 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 						//cerr << "below row: " << row << endl;
 						handle_hline_below(rowinfo[row], cellinfo[row]);
 					}
-				} else if (t.cs() == "cline") {
+				} else if (t.cs() == "cline" || t.cs() == "cmidrule") {
+					if (t.cs() == "cmidrule")
+						booktabs = true;
 					string arg = p1.verbatim_item();
-					//cerr << "read cline arg: '" << arg << "'\n";
+					//cerr << "read " << t.cs() << " arg: '" << arg << "'\n";
-					vector<string> t;
+					vector<string> cols;
-					split(arg, t, '-');
+					split(arg, cols, '-');
-					t.resize(2);
+					cols.resize(2);
-					size_t from = convert<unsigned int>(t[0]);
+					size_t from = convert<unsigned int>(cols[0]);
 					if (from == 0)
 						cerr << "Could not parse "
-							"cline start column."
+						     << t.cs() << " start column."
 						     << endl;
 					else
 						// 1 based index -> 0 based
 						--from;
 					if (from >= colinfo.size()) {
-						cerr << "cline starts at non "
+						cerr << t.cs() << " starts at "
-							"existing column "
+							"non existing column "
 						     << (from + 1) << endl;
 						from = colinfo.size() - 1;
 					}
-					size_t to = convert<unsigned int>(t[1]);
+					size_t to = convert<unsigned int>(cols[1]);
 					if (to == 0)
 						cerr << "Could not parse "
-							"cline end column."
+						     << t.cs() << " end column."
 						     << endl;
 					else
 						// 1 based index -> 0 based
 						--to;
 					if (to >= colinfo.size()) {
-						cerr << "cline ends at non "
+						cerr << t.cs() << " ends at "
-							"existing column "
+							"non existing column "
 						     << (to + 1) << endl;
 						to = colinfo.size() - 1;
 					}
@@ -900,38 +974,74 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 							cellinfo[row][col].bottomline = true;
 						}
 					}
+				} else if (t.cs() == "addlinespace") {
+					booktabs = true;
+					string const opt = p.next_token().cat() == catBegin ?
+							p.verbatim_item() : string();
+					if (i == 0) {
+						if (opt.empty())
+							rowinfo[row].top_space = "default";
+						else
+							rowinfo[row].top_space = translate_len(opt);
+					} else if (rowinfo[row].bottomline) {
+						if (opt.empty())
+							rowinfo[row].bottom_space = "default";
+						else
+							rowinfo[row].bottom_space = translate_len(opt);
+					} else {
+						if (opt.empty())
+							rowinfo[row].interline_space = "default";
+						else
+							rowinfo[row].interline_space = translate_len(opt);
+					}
 				} else if (t.cs() == "endhead") {
-					if (i > 0)
+					if (i == 0)
+						endhead.empty = true;
+					else
 						rowinfo[row].type = LT_HEAD;
 					for (int r = row - 1; r >= 0; --r) {
 						if (rowinfo[r].type != LT_NORMAL)
 							break;
 						rowinfo[r].type = LT_HEAD;
+						endhead.empty = false;
 					}
+					endhead.set = true;
 				} else if (t.cs() == "endfirsthead") {
-					if (i > 0)
+					if (i == 0)
+						endfirsthead.empty = true;
+					else
 						rowinfo[row].type = LT_FIRSTHEAD;
 					for (int r = row - 1; r >= 0; --r) {
 						if (rowinfo[r].type != LT_NORMAL)
 							break;
 						rowinfo[r].type = LT_FIRSTHEAD;
+						endfirsthead.empty = false;
 					}
+					endfirsthead.set = true;
 				} else if (t.cs() == "endfoot") {
-					if (i > 0)
+					if (i == 0)
+						endfoot.empty = true;
+					else
 						rowinfo[row].type = LT_FOOT;
 					for (int r = row - 1; r >= 0; --r) {
 						if (rowinfo[r].type != LT_NORMAL)
 							break;
 						rowinfo[r].type = LT_FOOT;
+						endfoot.empty = false;
 					}
+					endfoot.set = true;
 				} else if (t.cs() == "endlastfoot") {
-					if (i > 0)
+					if (i == 0)
+						endlastfoot.empty = true;
+					else
 						rowinfo[row].type = LT_LASTFOOT;
 					for (int r = row - 1; r >= 0; --r) {
 						if (rowinfo[r].type != LT_NORMAL)
 							break;
 						rowinfo[r].type = LT_LASTFOOT;
+						endlastfoot.empty = false;
 					}
+					endlastfoot.set = true;
 				} else if (t.cs() == "newpage") {
 					if (i == 0) {
 						if (row > 0)
@@ -950,6 +1060,48 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 			}
 		}
 
+		// LyX ends headers and footers always with \tabularnewline.
+		// This causes one additional row in the output.
+		// If the last row of a header/footer is empty, we can work
+		// around that by removing it.
+		if (row > 1) {
+			RowInfo test = rowinfo[row-1];
+			test.type = LT_NORMAL;
+			if (lines[row-1].empty() && !test.special()) {
+				switch (rowinfo[row-1].type) {
+				case LT_FIRSTHEAD:
+					if (rowinfo[row].type != LT_FIRSTHEAD &&
+					    rowinfo[row-2].type == LT_FIRSTHEAD)
+						deletelastrow = true;
+					break;
+				case LT_HEAD:
+					if (rowinfo[row].type != LT_HEAD &&
+					    rowinfo[row-2].type == LT_HEAD)
+						deletelastrow = true;
+					break;
+				case LT_FOOT:
+					if (rowinfo[row].type != LT_FOOT &&
+					    rowinfo[row-2].type == LT_FOOT)
+						deletelastrow = true;
+					break;
+				case LT_LASTFOOT:
+					if (rowinfo[row].type != LT_LASTFOOT &&
+					    rowinfo[row-2].type == LT_LASTFOOT)
+						deletelastrow = true;
+					break;
+				case LT_NORMAL:
+					break;
+				}
+			}
+		}
+
+		if (deletelastrow) {
+			lines.erase(lines.begin() + (row - 1));
+			rowinfo.erase(rowinfo.begin() + (row - 1));
+			cellinfo.erase(cellinfo.begin() + (row - 1));
+			continue;
+		}
+
 		// split into cells
 		vector<string> cells;
 		split(lines[row], cells, TAB);
@@ -1010,7 +1162,8 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 					cellinfo[row][col].align = 'c';
 				}
 
-			} else if (col == 0 && is_long_tabular &&
+			} else if (col == 0 && colinfo.size() > 1 &&
+			           is_long_tabular &&
 			           p.next_token().cs() == "caption") {
 				// longtable caption support in LyX is a hack:
 				// Captions require a row of their own with
@@ -1018,6 +1171,18 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 				// one multicolumn cell. The contents of that
 				// cell must contain exactly one caption inset
 				// and nothing else.
+				// LyX outputs all caption rows as first head,
+				// so we must not set the caption flag for
+				// captions not in the first head.
+				// Fortunately, the caption flag is only needed
+				// for tables with more than one column.
+				bool usecaption = (rowinfo[row].type == LT_NORMAL ||
+				                   rowinfo[row].type == LT_FIRSTHEAD);
+				for (size_t r = 0; r < row && usecaption; ++r)
+					if (rowinfo[row].type != LT_NORMAL &&
+					    rowinfo[row].type != LT_FIRSTHEAD)
+						usecaption = false;
+				if (usecaption) {
 					rowinfo[row].caption = true;
 					for (size_t c = 1; c < cells.size(); ++c) {
 						if (!cells[c].empty()) {
@@ -1032,13 +1197,19 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 					cells.resize(1);
 					cellinfo[row][col].align      = colinfo[col].align;
 					cellinfo[row][col].multi      = CELL_BEGIN_OF_MULTICOLUMN;
+				} else {
+					cellinfo[row][col].leftlines  = colinfo[col].leftlines;
+					cellinfo[row][col].rightlines = colinfo[col].rightlines;
+					cellinfo[row][col].align      = colinfo[col].align;
+				}
 				ostringstream os;
 				parse_text_in_inset(p, os, FLAG_CELL, false, context);
 				cellinfo[row][col].content += os.str();
+				if (usecaption) {
 					// add dummy multicolumn cells
 					for (size_t c = 1; c < colinfo.size(); ++c)
 						cellinfo[row][c].multi = CELL_PART_OF_MULTICOLUMN;
-
+				}
 			} else {
 				cellinfo[row][col].leftlines  = colinfo[col].leftlines;
 				cellinfo[row][col].rightlines = colinfo[col].rightlines;
@@ -1060,6 +1231,8 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 					cellinfo[row - 1][col].bottomline = true;
 			rowinfo.pop_back();
 		}
+
+		++row;
 	}
 
 	// Now we have the table structure and content in rowinfo, colinfo
@@ -1106,15 +1279,33 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 		}
 	}
 
+	if (booktabs)
+		preamble.registerAutomaticallyLoadedPackage("booktabs");
+	if (is_long_tabular)
+		preamble.registerAutomaticallyLoadedPackage("longtable");
+
 	//cerr << "// output what we have\n";
 	// output what we have
 	os << "\n<lyxtabular version=\"3\" rows=\"" << rowinfo.size()
 	   << "\" columns=\"" << colinfo.size() << "\">\n";
 	os << "<features"
 	   << write_attribute("rotate", false)
+	   << write_attribute("booktabs", booktabs)
 	   << write_attribute("islongtable", is_long_tabular);
-	if (!is_long_tabular)
+	if (is_long_tabular) {
-		os << write_attribute("tabularvalignment", tabularvalignment);
+		os << write_attribute("firstHeadTopDL", endfirsthead.topDL)
+		   << write_attribute("firstHeadBottomDL", endfirsthead.bottomDL)
+		   << write_attribute("firstHeadEmpty", endfirsthead.empty)
+		   << write_attribute("headTopDL", endhead.topDL)
+		   << write_attribute("headBottomDL", endhead.bottomDL)
+		   << write_attribute("footTopDL", endfoot.topDL)
+		   << write_attribute("footBottomDL", endfoot.bottomDL)
+		   << write_attribute("lastFootTopDL", endlastfoot.topDL)
+		   << write_attribute("lastFootBottomDL", endlastfoot.bottomDL)
+		   << write_attribute("lastFootEmpty", endlastfoot.empty);
+	} else
+		os << write_attribute("tabularvalignment", tabularvalignment)
+		   << write_attribute("tabularwidth", tabularwidth);
 	os << ">\n";
 
 	//cerr << "// after header\n";
@@ -1131,6 +1322,9 @@ void handle_tabular(Parser & p, ostream & os, bool is_long_tabular,
 
 	for (size_t row = 0; row < rowinfo.size(); ++row) {
 		os << "<row"
+		   << write_attribute("topspace", rowinfo[row].top_space)
+		   << write_attribute("bottomspace", rowinfo[row].bottom_space)
+		   << write_attribute("interlinespace", rowinfo[row].interline_space)
 		   << write_attribute("endhead",
 				      rowinfo[row].type == LT_HEAD)
 		   << write_attribute("endfirsthead",

src/tex2lyx/test/box-color-size-space-align.tex

@@ -87,6 +87,7 @@ blabla \makebox[3cm][l]{makebox 3} blabla
 \begin{picture}(8,6)
 \put(0,0){\makebox(0,0)[tr]{AAA}}
 \put(8,0){\makebox(0,0){BBB}}
+\put(0,8){\framebox(0,0){x}}
 \put(1,0){\line(1,0){6}}   
 \end{picture}
 

src/tex2lyx/test/test-insets.tex

@@ -114,8 +114,19 @@ M., \& Rasio, F.~A. 2004, ApJ, 604, 632\end{thebibliography}
 \section{Input files\index{Input files}}
 
 We can input files too, like this \input{DummyDocument}, or with the include
-variant \include{DummyDocument} % unfortunately, including the doc twice
+variant \include{DummyDocument} % unfortunately, including the doc twice generates a multiply defined label
-% generates a multiply defined label
+
+We can also import chess diagrams:
+
+\loadgame{../../../lib/examples/iecc05}\showboard
+
+Spreadsheets:
+
+\def\inputGnumericTable{}\input{../../../lib/examples/longsheet.gnumeric}
+
+and PDF pages:
+
+\includepdf[pages=-,angle=22,origin=Bl,width=5cm,height=40mm,keepaspectratio]{../../../lib/examples/beamer-icsi-logo}
 
 If you prefer verbatim input, you can choose
 between~\verbatiminput{foo} or~\verbatiminput*{foo}.
@@ -258,6 +269,97 @@ Lots of lines& like this.\\
 Lots of lines& like this.
 \end{longtable}
 
+From bug 7412 another example with more captions (can currently not produced in LyX):
+\begin{longtable}{|l|l|}
+\caption{A long table}
+\endfirsthead
+\caption{A long table -- continued}
+\endhead
+\multicolumn{2}{r}{{Continued on next page}}
+\tabularnewline
+\endfoot
+\endlastfoot
+\hline
+\multicolumn{1}{|c|}{Something} & \multicolumn{1}{c|}{Description}\tabularnewline
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline Lots of lines& like this.\\
+\hline
+\end{longtable}
+
+A table*:
+
+\begin{tabular*} % some comment
+{0.8\columnwidth}[b]{lr}
+two\\
+lonely&lines
+\end{tabular*}
+
+A booktabs table:
+
+\begin{table}[h]
+\caption{\label{tab:Special-booktabs-table}Special booktabs-table}
+
+
+\centering{}%
+\begin{tabular}{cccc}
+\toprule 
+System & Chip\,1 & \multicolumn{2}{c}{Chip\,2}\tabularnewline
+\cmidrule(r){2-2}\cmidrule(l){3-4}\morecmidrules \cmidrule{2-4}Detector
+thickness in \textmu{}m & 300 & 300 & 700\tabularnewline
+\midrule 
+Edge angle in \textdegree{} & 3.55 & 2.71 & 7.99\tabularnewline
+\addlinespace
+Spatial resolution in \textmu{}m & 4.26 & 10.17 & 10.56\tabularnewline
+\addlinespace
+MTF at $f_{\mathrm{max}}$ & 0.53 & 0.37 & 0.39\tabularnewline
+\midrule 
+\morecmidrules \cmidrule{3-4}LSF-spatial resolution &  &  & \tabularnewline
+in \textmu{}m & 129.7 & 52.75 & 50.78\tabularnewline
+in \% of pixel size & 76.3 & 95.9 & 92.3\tabularnewline
+\bottomrule
+\end{tabular}
+\end{table}
+
+
 \section{Macros}
 
 LyX supports several kinds of macros:

src/tex2lyx/test/test-structure.tex

@@ -16,6 +16,8 @@
 \newcommand{\noun}[1]{\textsc{#1}}
 %% Because html converters don't know tabularnewline
 \providecommand{\tabularnewline}{\\}
+\newcommand{\lyxadded}[3]{#3}
+\newcommand{\lyxdeleted}[3]{}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Textclass specific LaTeX commands.
 \newenvironment{lyxlist}[1]
@@ -87,6 +89,12 @@ This causes the \strong{logikalmkup} module to be loaded.
 An environment
 \end{quote}
 
+We also support change tracking:
+\lyxadded{Hans Wurst}{Sun Nov  6 10:39:39 2011}{Added text}
+some parts remain
+\lyxdeleted{Hans Wurst}{Sun Nov  6 10:39:55 2011}{This was the original text}
+some parts remain
+
 \section*{A starred section for floats}
 
 \begin{figure}
@@ -278,6 +286,7 @@ labelings:
 \begin{lyxlist}{00.00.0000}
 \item [label~1] first item
 \item [label~2] second item
+\item [{$\left[\textrm{ }\right]^{x}$}] Label with space, math and ] in it
 \end{lyxlist}
 and bibliography:
 \begin{thebibliography}{9}

src/tex2lyx/tex2lyx.1in

@@ -206,8 +206,6 @@ should thus be conserved in printed documents, although it will not of
 course show up in the LyX window. Check Document->Settings->LaTeX Preamble to see the result.
 .SS "What tex2lyx Can't Handle --- But it's \s-1OK\s0"
 .IP "\(bu" 4
-tabular* tables
-.IP "\(bu" 4
 some spacing commands (\f(CW\ehspace\fR, \f(CW\epagebreak\fR and \f(CW\elinebreak\fR)
 .IP "\(bu" 4
 \f(CW\ecentering\fR, \f(CW\eraggedleft\fR, \f(CW\eraggedright\fR

src/tex2lyx/tex2lyx.cpp

@@ -21,6 +21,7 @@
 #include "LayoutFile.h"
 #include "LayoutModuleList.h"
 #include "ModuleList.h"
+#include "Preamble.h"
 #include "TextClass.h"
 
 #include "support/convert.h"
@@ -342,6 +343,7 @@ bool checkModule(string const & name, bool command)
 
 bool noweb_mode = false;
 bool pdflatex = false;
+bool xetex = false;
 bool roundtrip = false;
 
 
@@ -364,12 +366,19 @@ void read_command(Parser & p, string command, CommandMap & commands)
 			string const arg = p.getArg('{', '}');
 			if (arg == "translate")
 				arguments.push_back(required);
+			else if (arg == "group")
+				arguments.push_back(req_group);
 			else if (arg == "item")
 				arguments.push_back(item);
+			else if (arg == "displaymath")
+				arguments.push_back(displaymath);
 			else
 				arguments.push_back(verbatim);
 		} else {
-			p.getArg('[', ']');
+			string const arg = p.getArg('[', ']');
+			if (arg == "group")
+				arguments.push_back(opt_group);
+			else
 				arguments.push_back(optional);
 		}
 	}
@@ -649,7 +658,7 @@ namespace {
  *  You must ensure that \p parentFilePath is properly set before calling
  *  this function!
  */
-void tex2lyx(idocstream & is, ostream & os, string encoding)
+bool tex2lyx(idocstream & is, ostream & os, string encoding)
 {
 	// Set a sensible default encoding.
 	// This is used until an encoding command is found.
@@ -657,18 +666,17 @@ void tex2lyx(idocstream & is, ostream & os, string encoding)
 	// since latin1 does not cause an iconv error if the actual encoding
 	// is different (bug 7509).
 	if (encoding.empty()) {
-		if (h_inputencoding == "auto")
+		if (preamble.inputencoding() == "auto")
 			encoding = "latin1";
 		else
-			encoding = h_inputencoding;
+			encoding = preamble.inputencoding();
 	}
 
 	Parser p(is);
 	p.setEncoding(encoding);
 	//p.dump();
 
-	ostringstream ps;
+	preamble.parse(p, documentclass, textclass);
-	parse_preamble(p, ps, documentclass, textclass);
 
 	active_environments.push_back("document");
 	Context context(true, textclass);
@@ -682,16 +690,16 @@ void tex2lyx(idocstream & is, ostream & os, string encoding)
 	active_environments.pop_back();
 
 	// We know the used modules only after parsing the full text
-	ostringstream ms;
 	if (!used_modules.empty()) {
-		ms << "\\begin_modules\n";
 		LayoutModuleList::const_iterator const end = used_modules.end();
 		LayoutModuleList::const_iterator it = used_modules.begin();
 		for (; it != end; it++)
-			ms << *it << '\n';
+			preamble.addModule(*it);
-		ms << "\\end_modules\n";
+	}
+	if (!preamble.writeLyXHeader(os, !active_environments.empty())) {
+		cerr << "Could write LyX file header." << endl;
+		return false;
 	}
-	os << subst(ps.str(), modules_placeholder, ms.str());
 
 	ss.seekg(0);
 	os << ss.str();
@@ -702,6 +710,7 @@ void tex2lyx(idocstream & is, ostream & os, string encoding)
 		parsertest << p.get_token().asInput();
 	// <origfile> and parsertest.tex should now have identical content
 #endif
+	return true;
 }
 
 
@@ -719,9 +728,9 @@ bool tex2lyx(FileName const & infilename, ostream & os, string const & encoding)
 	}
 	string const oldParentFilePath = parentFilePath;
 	parentFilePath = onlyPath(infilename.absFileName());
-	tex2lyx(is, os, encoding);
+	bool retval = tex2lyx(is, os, encoding);
 	parentFilePath = oldParentFilePath;
-	return true;
+	return retval;
 }
 
 } // anonymous namespace

src/tex2lyx/tex2lyx.h

@@ -43,19 +43,10 @@ public:
 	void setName(std::string const & name) { name_ = name; }
 };
 
-/// in preamble.cpp
-void parse_preamble(Parser & p, std::ostream & os, 
-	std::string const & forceclass, TeX2LyXDocClass & tc);
 /// Translate babel language name to LyX language name
 extern std::string babel2lyx(std::string const & language);
-/// translate color name to LyX color code
+/// Translate basic color name or RGB color in LaTeX syntax to LyX color code
-extern std::string color2code(std::string const & name);
+extern std::string rgbcolor2code(std::string const & name);
-
-/// used packages with options
-extern std::map<std::string, std::vector<std::string> > used_packages;
-extern const char * const modules_placeholder;
-extern std::string h_inputencoding;
-extern std::string h_paragraph_separation;
 
 /// in text.cpp
 std::string translate_len(std::string const &);
@@ -80,8 +71,8 @@ void parse_math(Parser & p, std::ostream & os, unsigned flags, mode_type mode);
 
 
 /// in table.cpp
-void handle_tabular(Parser & p, std::ostream & os, bool is_long_tabular,
+void handle_tabular(Parser & p, std::ostream & os, std::string const & name,
-		    Context & context);
+                    std::string const & width, Context & context);
 
 
 /// in tex2lyx.cpp
@@ -93,6 +84,7 @@ std::string join(std::vector<std::string> const & input,
 	char const * delim);
 
 bool is_math_env(std::string const & name);
+bool is_display_math_env(std::string const & name);
 char const * const * is_known(std::string const &, char const * const *);
 
 /*!
@@ -124,9 +116,12 @@ std::string active_environment();
 
 enum ArgumentType {
 	required,
+	req_group,
 	verbatim,
 	item,
-	optional
+	optional,
+	opt_group,
+	displaymath,
 };
 
 class FullCommand {
@@ -167,6 +162,8 @@ extern FullEnvironmentMap possible_textclass_environments;
 extern bool noweb_mode;
 /// Did we recognize any pdflatex-only construct?
 extern bool pdflatex;
+/// Did we recognize any xetex-only construct?
+extern bool xetex;
 /// LyX format that is created by tex2lyx
 int const LYX_FORMAT = 413;
 

src/tex2lyx/text.cpp

@@ -19,14 +19,17 @@
 #include "Context.h"
 #include "Encoding.h"
 #include "FloatList.h"
+#include "LaTeXPackages.h"
 #include "Layout.h"
 #include "Length.h"
+#include "Preamble.h"
 
 #include "support/lassert.h"
 #include "support/convert.h"
 #include "support/FileName.h"
 #include "support/filetools.h"
 #include "support/lstrings.h"
+#include "support/lyxtime.h"
 
 #include <algorithm>
 #include <iostream>
@@ -160,7 +163,12 @@ char const * const known_old_font_families[] = { "rm", "sf", "tt", 0};
 char const * const known_font_families[] = { "rmfamily", "sffamily",
 "ttfamily", 0};
 
-/// the same as known_old_font_families and known_font_families with .lyx names
+/// LaTeX names for font family changing commands
+char const * const known_text_font_families[] = { "textrm", "textsf",
+"texttt", 0};
+
+/// The same as known_old_font_families, known_font_families and
+/// known_text_font_families with .lyx names
 char const * const known_coded_font_families[] = { "roman", "sans",
 "typewriter", 0};
 
@@ -170,7 +178,11 @@ char const * const known_old_font_series[] = { "bf", 0};
 /// LaTeX names for font series
 char const * const known_font_series[] = { "bfseries", "mdseries", 0};
 
-/// the same as known_old_font_series and known_font_series with .lyx names
+/// LaTeX names for font series changing commands
+char const * const known_text_font_series[] = { "textbf", "textmd", 0};
+
+/// The same as known_old_font_series, known_font_series and
+/// known_text_font_series with .lyx names
 char const * const known_coded_font_series[] = { "bold", "medium", 0};
 
 /// LaTeX 2.09 names for font shapes
@@ -180,10 +192,23 @@ char const * const known_old_font_shapes[] = { "it", "sl", "sc", 0};
 char const * const known_font_shapes[] = { "itshape", "slshape", "scshape",
 "upshape", 0};
 
-/// the same as known_old_font_shapes and known_font_shapes with .lyx names
+/// LaTeX names for font shape changing commands
+char const * const known_text_font_shapes[] = { "textit", "textsl", "textsc",
+"textup", 0};
+
+/// The same as known_old_font_shapes, known_font_shapes and
+/// known_text_font_shapes with .lyx names
 char const * const known_coded_font_shapes[] = { "italic", "slanted",
 "smallcaps", "up", 0};
 
+/// Known special characters which need skip_spaces_braces() afterwards
+char const * const known_special_chars[] = {"ldots", "lyxarrow",
+"textcompwordmark", "slash", 0};
+
+/// the same as known_special_chars with .lyx names
+char const * const known_coded_special_chars[] = {"ldots{}", "menuseparator",
+"textcompwordmark{}", "slash{}", 0};
+
 /*!
  * Graphics file extensions known by the dvips driver of the graphics package.
  * These extensions are used to complete the filename of an included
@@ -668,10 +693,14 @@ void parse_arguments(string const & command,
 	for (size_t i = 0; i < no_arguments; ++i) {
 		switch (template_arguments[i]) {
 		case required:
+		case req_group:
 			// This argument contains regular LaTeX
 			handle_ert(os, ert + '{', context);
 			eat_whitespace(p, os, context, false);
+			if (template_arguments[i] == required)
 				parse_text(p, os, FLAG_ITEM, outer, context);
+			else
+				parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 			ert = "}";
 			break;
 		case item:
@@ -683,11 +712,13 @@ void parse_arguments(string const & command,
 			else
 				ert += p.verbatim_item();
 			break;
+		case displaymath:
 		case verbatim:
 			// This argument may contain special characters
 			ert += '{' + p.verbatim_item() + '}';
 			break;
 		case optional:
+		case opt_group:
 			// true because we must not eat whitespace
 			// if an optional arg follows we must not strip the
 			// brackets from this one
@@ -792,7 +823,7 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
 		latex_width = p.verbatim_item();
 	// if e.g. only \ovalbox{content} was used, set the width to 1\columnwidth
 	// as this is LyX's standard for such cases (except for makebox)
-	// \framebox is special and handled below
+	// \framebox is more special and handled below
 	if (latex_width.empty() && inner_type != "makebox"
 		&& outer_type != "framebox")
 		latex_width = "1\\columnwidth";
@@ -927,6 +958,7 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
 			     (outer_type == "minipage" && inner_type == "shaded") ||
 			     (outer_type == "parbox" && inner_type == "shaded")) {
 			os << "Shaded\n";
+			preamble.registerAutomaticallyLoadedPackage("color");
 		} else if (outer_type == "doublebox")
 			os << "Doublebox\n";
 		else if (outer_type.empty())
@@ -1133,7 +1165,8 @@ void parse_unknown_environment(Parser & p, string const & name, ostream & os,
 
 
 void parse_environment(Parser & p, ostream & os, bool outer,
-                       string & last_env, Context & parent_context)
+                       string & last_env, bool & title_layout_found,
+                       Context & parent_context)
 {
 	Layout const * newlayout;
 	InsetLayout const * newinsetlayout = 0;
@@ -1149,13 +1182,24 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 		parse_math(p, os, FLAG_END, MATH_MODE);
 		os << "\\end{" << name << "}";
 		end_inset(os);
+		if (is_display_math_env(name)) {
+			// Prevent the conversion of a line break to a space
+			// (bug 7668). This does not change the output, but
+			// looks ugly in LyX.
+			eat_whitespace(p, os, parent_context, false);
+		}
 	}
 
-	else if (name == "tabular" || name == "longtable") {
+	else if (unstarred_name == "tabular" || name == "longtable") {
 		eat_whitespace(p, os, parent_context, false);
+		string width = "0pt";
+		if (name == "tabular*") {
+			width = lyx::translate_len(p.getArg('{', '}'));
+			eat_whitespace(p, os, parent_context, false);
+		}
 		parent_context.check_layout(os);
 		begin_inset(os, "Tabular ");
-		handle_tabular(p, os, name == "longtable", parent_context);
+		handle_tabular(p, os, name, width, parent_context);
 		end_inset(os);
 		p.skip_spaces();
 	}
@@ -1176,6 +1220,15 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 			float_type = "";
 		if (!opt.empty())
 			os << "placement " << opt << '\n';
+		if (contains(opt, "H"))
+                	preamble.registerAutomaticallyLoadedPackage("float");
+		else {
+			Floating const & fl = parent_context.textclass.floats()
+				.getType(unstarred_name);
+		        if (!fl.floattype().empty() && fl.usesFloatPkg())
+                		preamble.registerAutomaticallyLoadedPackage("float");
+		}
+
 		os << "wide " << convert<string>(is_starred)
 		   << "\nsideways false"
 		   << "\nstatus open\n\n";
@@ -1287,6 +1340,8 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 		parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
 		end_inset(os);
 		p.skip_spaces();
+		if (!preamble.notefontcolor().empty())
+			preamble.registerAutomaticallyLoadedPackage("color");
 	}
 
 	else if (name == "framed" || name == "shaded") {
@@ -1298,6 +1353,8 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 	else if (name == "lstlisting") {
 		eat_whitespace(p, os, parent_context, false);
 		// FIXME handle listings with parameters
+		//       If this is added, don't forgot to handle the
+		//       automatic color package loading
 		if (p.hasOpt())
 			parse_unknown_environment(p, name, os, FLAG_END,
 			                          outer, parent_context);
@@ -1335,12 +1392,16 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 			parent_context.add_extra_stuff("\\align center\n");
 		else if (name == "singlespace")
 			parent_context.add_extra_stuff("\\paragraph_spacing single\n");
-		else if (name == "onehalfspace")
+		else if (name == "onehalfspace") {
 			parent_context.add_extra_stuff("\\paragraph_spacing onehalf\n");
-		else if (name == "doublespace")
+			preamble.registerAutomaticallyLoadedPackage("setspace");
+		} else if (name == "doublespace") {
 			parent_context.add_extra_stuff("\\paragraph_spacing double\n");
-		else if (name == "spacing")
+			preamble.registerAutomaticallyLoadedPackage("setspace");
+		} else if (name == "spacing") {
 			parent_context.add_extra_stuff("\\paragraph_spacing other " + p.verbatim_item() + "\n");
+			preamble.registerAutomaticallyLoadedPackage("setspace");
+		}
 		parse_text(p, os, FLAG_END, outer, parent_context);
 		// Just in case the environment is empty
 		parent_context.extra_stuff.erase();
@@ -1450,6 +1511,11 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 		context.check_end_deeper(os);
 		parent_context.new_paragraph(os);
 		p.skip_spaces();
+		if (!title_layout_found)
+			title_layout_found = newlayout->intitle;
+		set<string> const & req = newlayout->requires();
+		for (set<string>::const_iterator it = req.begin(); it != req.end(); it++)
+			preamble.registerAutomaticallyLoadedPackage(*it);
 	}
 
 	// The single '=' is meant here.
@@ -1832,13 +1898,15 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 {
 	Layout const * newlayout = 0;
 	InsetLayout const * newinsetlayout = 0;
+	char const * const * where = 0;
 	// Store the latest bibliographystyle and nocite{*} option
 	// (needed for bibtex inset)
 	string btprint;
 	string bibliographystyle;
-	bool const use_natbib = used_packages.find("natbib") != used_packages.end();
+	bool const use_natbib = preamble.isPackageUsed("natbib");
-	bool const use_jurabib = used_packages.find("jurabib") != used_packages.end();
+	bool const use_jurabib = preamble.isPackageUsed("jurabib");
 	string last_env;
+	bool title_layout_found = false;
 	while (p.good()) {
 		Token const & t = p.get_token();
 
@@ -1882,7 +1950,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			context.check_layout(os);
 			begin_inset(os, "Formula ");
 			Token const & n = p.get_token();
-			if (n.cat() == catMath && outer) {
+			bool const display(n.cat() == catMath && outer);
+			if (display) {
 				// TeX's $$...$$ syntax for displayed math
 				os << "\\[";
 				parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
@@ -1896,6 +1965,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 				os << '$';
 			}
 			end_inset(os);
+			if (display) {
+				// Prevent the conversion of a line break to a
+				// space (bug 7668). This does not change the
+				// output, but looks ugly in LyX.
+				eat_whitespace(p, os, context, false);
+			}
 		}
 
 		else if (t.cat() == catSuper || t.cat() == catSub)
@@ -2017,8 +2092,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 				os << t.cs();
 		}
 
-		else if (t.cat() == catBegin &&
+		else if (t.cat() == catBegin) {
-			 p.next_token().cat() == catEnd) {
+			Token const next = p.next_token();
+			Token const end = p.next_next_token();
+			if (next.cat() == catEnd) {
 			// {}
 			Token const prev = p.prev_token();
 			p.get_token();
@@ -2028,14 +2105,19 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 				; // ignore it in {}`` or -{}-
 			else
 				handle_ert(os, "{}", context);
-
+			} else if (next.cat() == catEscape &&
-		}
+			           is_known(next.cs(), known_quotes) &&
-
+			           end.cat() == catEnd) {
-		else if (t.cat() == catBegin) {
+				// Something like {\textquoteright} (e.g.
+				// from writer2latex). LyX writes
+				// \textquoteright{}, so we may skip the
+				// braces here for better readability.
+				parse_text_snippet(p, os, FLAG_BRACE_LAST,
+				                   outer, context);
+			} else {
 			context.check_layout(os);
 			// special handling of font attribute changes
 			Token const prev = p.prev_token();
-			Token const next = p.next_token();
 			TeXFont const oldFont = context.font;
 			if (next.character() == '[' ||
 			    next.character() == ']' ||
@@ -2110,6 +2192,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 				handle_ert(os, "}", context);
 				}
 			}
+		}
 
 		else if (t.cat() == catEnd) {
 			if (flags & FLAG_BRACE_LAST) {
@@ -2142,10 +2225,15 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			parse_math(p, os, FLAG_EQUATION, MATH_MODE);
 			os << "\\]";
 			end_inset(os);
+			// Prevent the conversion of a line break to a space
+			// (bug 7668). This does not change the output, but
+			// looks ugly in LyX.
+			eat_whitespace(p, os, context, false);
 		}
 
 		else if (t.cs() == "begin")
-			parse_environment(p, os, outer, last_env, context);
+			parse_environment(p, os, outer, last_env,
+			                  title_layout_found, context);
 
 		else if (t.cs() == "end") {
 			if (flags & FLAG_END) {
@@ -2166,7 +2254,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 				// FIXME: This swallows comments, but we cannot use
 				//        eat_whitespace() since we must not output
 				//        anything before the item.
-				s = p.getArg('[', ']');
+				p.skip_spaces(true);
+				s = p.verbatimOption();
 			} else
 				p.skip_spaces(false);
 			context.set_item();
@@ -2216,7 +2305,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 		else if (t.cs() == "bibitem") {
 			context.set_item();
 			context.check_layout(os);
-			string label = convert_command_inset_arg(p.getArg('[', ']'));
+			eat_whitespace(p, os, context, false);
+			string label = convert_command_inset_arg(p.verbatimOption());
 			string key = convert_command_inset_arg(p.verbatim_item());
 			if (contains(label, '\\') || contains(key, '\\')) {
 				// LyX can't handle LaTeX commands in labels or keys
@@ -2231,8 +2321,53 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			}
 		}
 
-		else if (is_macro(p))
+		else if (is_macro(p)) {
+			// catch the case of \def\inputGnumericTable
+			bool macro = true;
+			if (t.cs() == "def") {
+				Token second = p.next_token();
+				if (second.cs() == "inputGnumericTable") {
+					p.pushPosition();
+					p.get_token();
+					skip_braces(p);
+					Token third = p.get_token();
+					p.popPosition();
+					if (third.cs() == "input") {
+						p.get_token();
+						skip_braces(p);
+						p.get_token();
+						string name = normalize_filename(p.verbatim_item());
+						string const path = getMasterFilePath();
+						// We want to preserve relative / absolute filenames,
+						// therefore path is only used for testing
+						if (!makeAbsPath(name, path).exists()) {
+							// The file extension is probably missing.
+							// Now try to find it out.
+							char const * const Gnumeric_formats[] = {"gnumeric"
+								"ods", "xls", 0};
+							string const Gnumeric_name =
+								find_file(name, path, Gnumeric_formats);
+							if (!Gnumeric_name.empty())
+								name = Gnumeric_name;
+						}
+						if (makeAbsPath(name, path).exists())
+							fix_relative_filename(name);
+						else
+							cerr << "Warning: Could not find file '"
+							     << name << "'." << endl;
+						context.check_layout(os);
+						begin_inset(os, "External\n\ttemplate ");
+						os << "GnumericSpreadsheet\n\tfilename "
+						   << name << "\n";
+						end_inset(os);
+						context.check_layout(os);
+						macro = false;
+					}
+				}
+			}
+			if (macro)
 				parse_macro(p, os, context);
+		}
 
 		else if (t.cs() == "noindent") {
 			p.skip_spaces();
@@ -2262,6 +2397,32 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			eat_whitespace(p, os, context, true);
 		}
 
+		// Must catch empty dates before findLayout is called below
+		else if (t.cs() == "date") {
+			string const date = p.verbatim_item();
+			if (date.empty())
+				preamble.suppressDate(true);
+			else {
+				preamble.suppressDate(false);
+				if (context.new_layout_allowed &&
+				    (newlayout = findLayout(context.textclass,
+				                            t.cs(), true))) {
+					// write the layout
+					output_command_layout(os, p, outer,
+							context, newlayout);
+					p.skip_spaces();
+					if (!title_layout_found)
+						title_layout_found = newlayout->intitle;
+					set<string> const & req = newlayout->requires();
+					for (set<string>::const_iterator it = req.begin();
+					     it != req.end(); it++)
+						preamble.registerAutomaticallyLoadedPackage(*it);
+				} else
+					handle_ert(os, "\\date{" + date + '}',
+							context);
+			}
+		}
+
 		// Starred section headings
 		// Must attempt to parse "Section*" before "Section".
 		else if ((p.next_token().asInput() == "*") &&
@@ -2271,6 +2432,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			p.get_token();
 			output_command_layout(os, p, outer, context, newlayout);
 			p.skip_spaces();
+			if (!title_layout_found)
+				title_layout_found = newlayout->intitle;
+			set<string> const & req = newlayout->requires();
+			for (set<string>::const_iterator it = req.begin(); it != req.end(); it++)
+				preamble.registerAutomaticallyLoadedPackage(*it);
 		}
 
 		// Section headings and the like
@@ -2279,6 +2445,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			// write the layout
 			output_command_layout(os, p, outer, context, newlayout);
 			p.skip_spaces();
+			if (!title_layout_found)
+				title_layout_found = newlayout->intitle;
+			set<string> const & req = newlayout->requires();
+			for (set<string>::const_iterator it = req.begin(); it != req.end(); it++)
+				preamble.registerAutomaticallyLoadedPackage(*it);
 		}
 
 		else if (t.cs() == "caption") {
@@ -2561,10 +2732,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 		}
 
 		else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
-			// FIXME: Somehow prevent title layouts if
+			if (title_layout_found) {
-			// "maketitle" was not found
 				// swallow this
 				skip_spaces_braces(p);
+			} else
+				handle_ert(os, t.asInput(), context);
 		}
 
 		else if (t.cs() == "tableofcontents") {
@@ -2601,50 +2773,20 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 				handle_ert(os, "\\listof{" + name + "}", context);
 		}
 
-		else if (t.cs() == "textrm")
+		else if ((where = is_known(t.cs(), known_text_font_families)))
 			parse_text_attributes(p, os, FLAG_ITEM, outer,
-					      context, "\\family",
+				context, "\\family", context.font.family,
-					      context.font.family, "roman");
+				known_coded_font_families[where - known_text_font_families]);
 
-		else if (t.cs() == "textsf")
+		else if ((where = is_known(t.cs(), known_text_font_series)))
 			parse_text_attributes(p, os, FLAG_ITEM, outer,
-					      context, "\\family",
+				context, "\\series", context.font.series,
-					      context.font.family, "sans");
+				known_coded_font_series[where - known_text_font_series]);
 
-		else if (t.cs() == "texttt")
+		else if ((where = is_known(t.cs(), known_text_font_shapes)))
 			parse_text_attributes(p, os, FLAG_ITEM, outer,
-					      context, "\\family",
+				context, "\\shape", context.font.shape,
-					      context.font.family, "typewriter");
+				known_coded_font_shapes[where - known_text_font_shapes]);
-
-		else if (t.cs() == "textmd")
-			parse_text_attributes(p, os, FLAG_ITEM, outer,
-					      context, "\\series",
-					      context.font.series, "medium");
-
-		else if (t.cs() == "textbf")
-			parse_text_attributes(p, os, FLAG_ITEM, outer,
-					      context, "\\series",
-					      context.font.series, "bold");
-
-		else if (t.cs() == "textup")
-			parse_text_attributes(p, os, FLAG_ITEM, outer,
-					      context, "\\shape",
-					      context.font.shape, "up");
-
-		else if (t.cs() == "textit")
-			parse_text_attributes(p, os, FLAG_ITEM, outer,
-					      context, "\\shape",
-					      context.font.shape, "italic");
-
-		else if (t.cs() == "textsl")
-			parse_text_attributes(p, os, FLAG_ITEM, outer,
-					      context, "\\shape",
-					      context.font.shape, "slanted");
-
-		else if (t.cs() == "textsc")
-			parse_text_attributes(p, os, FLAG_ITEM, outer,
-					      context, "\\shape",
-					      context.font.shape, "smallcaps");
 
 		else if (t.cs() == "textnormal" || t.cs() == "normalfont") {
 			context.check_layout(os);
@@ -2674,6 +2816,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 					parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 					context.check_layout(os);
 					os << "\n\\color inherit\n";
+					preamble.registerAutomaticallyLoadedPackage("color");
 			} else
 				// for custom defined colors
 				handle_ert(os, t.asInput() + "{" + color + "}", context);
@@ -2691,6 +2834,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 			context.check_layout(os);
 			os << "\n\\bar default\n";
+			preamble.registerAutomaticallyLoadedPackage("ulem");
 		}
 
 		else if (t.cs() == "sout") {
@@ -2699,6 +2843,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 			context.check_layout(os);
 			os << "\n\\strikeout default\n";
+			preamble.registerAutomaticallyLoadedPackage("ulem");
 		}
 
 		else if (t.cs() == "uuline" || t.cs() == "uwave" ||
@@ -2708,6 +2853,52 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 			context.check_layout(os);
 			os << "\n\\" << t.cs() << " default\n";
+			if (t.cs() == "uuline" || t.cs() == "uwave")
+				preamble.registerAutomaticallyLoadedPackage("ulem");
+		}
+
+		else if (t.cs() == "lyxadded" || t.cs() == "lyxdeleted") {
+			context.check_layout(os);
+			string name = p.getArg('{', '}');
+			string localtime = p.getArg('{', '}');
+			preamble.registerAuthor(name);
+			Author const & author = preamble.getAuthor(name);
+			// from_ctime() will fail if LyX decides to output the
+			// time in the text language. It might also use a wrong
+			// time zone (if the original LyX document was exported
+			// with a different time zone).
+			time_t ptime = from_ctime(localtime);
+			if (ptime == static_cast<time_t>(-1)) {
+				cerr << "Warning: Could not parse time `" << localtime
+				     << "´ for change tracking, using current time instead.\n";
+				ptime = current_time();
+			}
+			if (t.cs() == "lyxadded")
+				os << "\n\\change_inserted ";
+			else
+				os << "\n\\change_deleted ";
+			os << author.bufferId() << ' ' << ptime << '\n';
+			parse_text_snippet(p, os, FLAG_ITEM, outer, context);
+			bool dvipost    = LaTeXPackages::isAvailable("dvipost");
+			bool xcolorulem = LaTeXPackages::isAvailable("ulem") &&
+			                  LaTeXPackages::isAvailable("xcolor");
+			// No need to test for luatex, since luatex comes in
+			// two flavours (dvi and pdf), like latex, and those
+			// are detected by pdflatex.
+			if (pdflatex || xetex) {
+				if (xcolorulem) {
+					preamble.registerAutomaticallyLoadedPackage("ulem");
+					preamble.registerAutomaticallyLoadedPackage("xcolor");
+					preamble.registerAutomaticallyLoadedPackage("pdfcolmk");
+				}
+			} else {
+				if (dvipost) {
+					preamble.registerAutomaticallyLoadedPackage("dvipost");
+				} else if (xcolorulem) {
+					preamble.registerAutomaticallyLoadedPackage("ulem");
+					preamble.registerAutomaticallyLoadedPackage("xcolor");
+				}
+			}
 		}
 
 		else if (t.cs() == "phantom" || t.cs() == "hphantom" ||
@@ -2764,7 +2955,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 				//        about the empty paragraph.
 				context.new_paragraph(os);
 			}
-			if (h_paragraph_separation == "indent") {
+			if (preamble.indentParagraphs()) {
 				// we need to unindent, lest the line be too long
 				context.add_par_extra_stuff("\\noindent\n");
 			}
@@ -2795,7 +2986,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 		          is_known(p.next_token().cs(), known_phrases))) {
 			// LyX sometimes puts a \protect in front, so we have to ignore it
 			// FIXME: This needs to be changed when bug 4752 is fixed.
-			char const * const * where = is_known(
+			where = is_known(
 				t.cs() == "protect" ? p.get_token().cs() : t.cs(),
 				known_phrases);
 			context.check_layout(os);
@@ -2803,12 +2994,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			skip_spaces_braces(p);
 		}
 
-		else if (is_known(t.cs(), known_ref_commands)) {
+		else if ((where = is_known(t.cs(), known_ref_commands))) {
 			string const opt = p.getOpt();
 			if (opt.empty()) {
 				context.check_layout(os);
-				char const * const * where = is_known(t.cs(),
-					known_ref_commands);
 				begin_command_inset(os, "ref",
 					known_coded_ref_commands[where - known_ref_commands]);
 				os << "reference \""
@@ -2891,7 +3080,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 				p.get_token();
 			}
 			char argumentOrder = '\0';
-			vector<string> const & options = used_packages["jurabib"];
+			vector<string> const options =
+				preamble.getPackageOptions("jurabib");
 			if (find(options.begin(), options.end(),
 				      "natbiborder") != options.end())
 				argumentOrder = 'n';
@@ -2966,6 +3156,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			   << convert_command_inset_arg(p.verbatim_item())
 			   << "\"\n";
 			end_inset(os);
+			preamble.registerAutomaticallyLoadedPackage("nomencl");
 		}
 		
 		else if (t.cs() == "label") {
@@ -2983,6 +3174,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			os << "type \"idx\"\n";
 			end_inset(os);
 			skip_spaces_braces(p);
+			preamble.registerAutomaticallyLoadedPackage("makeidx");
+			if (preamble.use_indices() == "true")
+				preamble.registerAutomaticallyLoadedPackage("splitidx");
 		}
 
 		else if (t.cs() == "printnomenclature") {
@@ -3009,6 +3203,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 				os << "width \"" << width << '\"';
 			end_inset(os);
 			skip_spaces_braces(p);
+			preamble.registerAutomaticallyLoadedPackage("nomencl");
 		}
 
 		else if ((t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
@@ -3017,10 +3212,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			os << t.cs().substr(4) << '\n';
 			parse_text_in_inset(p, os, FLAG_ITEM, false, context);
 			end_inset(os);
+			if (t.cs() == "textsubscript")
+				preamble.registerAutomaticallyLoadedPackage("subscript");
 		}
 
-		else if (is_known(t.cs(), known_quotes)) {
+		else if ((where = is_known(t.cs(), known_quotes))) {
-			char const * const * where = is_known(t.cs(), known_quotes);
 			context.check_layout(os);
 			begin_inset(os, "Quotes ");
 			os << known_coded_quotes[where - known_quotes];
@@ -3032,9 +3228,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			skip_braces(p);
 		}
 
-		else if (is_known(t.cs(), known_sizes) &&
+		else if ((where = is_known(t.cs(), known_sizes)) &&
 			 context.new_layout_allowed) {
-			char const * const * where = is_known(t.cs(), known_sizes);
 			context.check_layout(os);
 			TeXFont const oldFont = context.font;
 			context.font.size = known_coded_sizes[where - known_sizes];
@@ -3042,10 +3237,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			eat_whitespace(p, os, context, false);
 		}
 
-		else if (is_known(t.cs(), known_font_families) &&
+		else if ((where = is_known(t.cs(), known_font_families)) &&
 			 context.new_layout_allowed) {
-			char const * const * where =
-				is_known(t.cs(), known_font_families);
 			context.check_layout(os);
 			TeXFont const oldFont = context.font;
 			context.font.family =
@@ -3054,10 +3247,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			eat_whitespace(p, os, context, false);
 		}
 
-		else if (is_known(t.cs(), known_font_series) &&
+		else if ((where = is_known(t.cs(), known_font_series)) &&
 			 context.new_layout_allowed) {
-			char const * const * where =
-				is_known(t.cs(), known_font_series);
 			context.check_layout(os);
 			TeXFont const oldFont = context.font;
 			context.font.series =
@@ -3066,10 +3257,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			eat_whitespace(p, os, context, false);
 		}
 
-		else if (is_known(t.cs(), known_font_shapes) &&
+		else if ((where = is_known(t.cs(), known_font_shapes)) &&
 			 context.new_layout_allowed) {
-			char const * const * where =
-				is_known(t.cs(), known_font_shapes);
 			context.check_layout(os);
 			TeXFont const oldFont = context.font;
 			context.font.shape =
@@ -3077,10 +3266,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			output_font_change(os, oldFont, context.font);
 			eat_whitespace(p, os, context, false);
 		}
-		else if (is_known(t.cs(), known_old_font_families) &&
+		else if ((where = is_known(t.cs(), known_old_font_families)) &&
 			 context.new_layout_allowed) {
-			char const * const * where =
-				is_known(t.cs(), known_old_font_families);
 			context.check_layout(os);
 			TeXFont const oldFont = context.font;
 			context.font.init();
@@ -3091,10 +3278,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			eat_whitespace(p, os, context, false);
 		}
 
-		else if (is_known(t.cs(), known_old_font_series) &&
+		else if ((where = is_known(t.cs(), known_old_font_series)) &&
 			 context.new_layout_allowed) {
-			char const * const * where =
-				is_known(t.cs(), known_old_font_series);
 			context.check_layout(os);
 			TeXFont const oldFont = context.font;
 			context.font.init();
@@ -3105,10 +3290,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			eat_whitespace(p, os, context, false);
 		}
 
-		else if (is_known(t.cs(), known_old_font_shapes) &&
+		else if ((where = is_known(t.cs(), known_old_font_shapes)) &&
 			 context.new_layout_allowed) {
-			char const * const * where =
-				is_known(t.cs(), known_old_font_shapes);
 			context.check_layout(os);
 			TeXFont const oldFont = context.font;
 			context.font.init();
@@ -3141,27 +3324,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			p.setEncoding(enc);
 		}
 
-		else if (t.cs() == "ldots") {
+		else if ((where = is_known(t.cs(), known_special_chars))) {
 			context.check_layout(os);
-			os << "\\SpecialChar \\ldots{}\n";
+			os << "\\SpecialChar \\"
-			skip_spaces_braces(p);
+			   << known_coded_special_chars[where - known_special_chars]
-		}
+			   << '\n';
-
-		else if (t.cs() == "lyxarrow") {
-			context.check_layout(os);
-			os << "\\SpecialChar \\menuseparator\n";
-			skip_spaces_braces(p);
-		}
-
-		else if (t.cs() == "textcompwordmark") {
-			context.check_layout(os);
-			os << "\\SpecialChar \\textcompwordmark{}\n";
-			skip_spaces_braces(p);
-		}
-
-		else if (t.cs() == "slash") {
-			context.check_layout(os);
-			os << "\\SpecialChar \\slash{}\n";
 			skip_spaces_braces(p);
 		}
 
@@ -3477,17 +3644,30 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), "");
 
 		else if (t.cs() == "framebox") {
+			if (p.next_token().character() == '(') {
+				//the syntax is: \framebox(x,y)[position]{content}
+				string arg = t.asInput();
+				arg += p.getFullParentheseArg();
+				arg += p.getFullOpt();
+				eat_whitespace(p, os, context, false);
+				handle_ert(os, arg + '{', context);
+				eat_whitespace(p, os, context, false);
+				parse_text(p, os, FLAG_ITEM, outer, context);
+				handle_ert(os, "}", context);
+			} else {
 				string special = p.getFullOpt();
 				special += p.getOpt();
-			parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), special);
+				parse_outer_box(p, os, FLAG_ITEM, outer,
+				                context, t.cs(), special);
+			}
 		}
 
 		//\makebox() is part of the picture environment and different from \makebox{}
 		//\makebox{} will be parsed by parse_box
 		else if (t.cs() == "makebox") {
-			string arg = t.asInput();
 			if (p.next_token().character() == '(') {
 				//the syntax is: \makebox(x,y)[position]{content}
+				string arg = t.asInput();
 				arg += p.getFullParentheseArg();
 				arg += p.getFullOpt();
 				eat_whitespace(p, os, context, false);
@@ -3512,8 +3692,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			skip_spaces_braces(p);
 		}
 
-		else if (is_known(t.cs(), known_spaces)) {
+		else if ((where = is_known(t.cs(), known_spaces))) {
-			char const * const * where = is_known(t.cs(), known_spaces);
 			context.check_layout(os);
 			begin_inset(os, "space ");
 			os << '\\' << known_coded_spaces[where - known_spaces]
@@ -3731,6 +3910,37 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			end_inset(os);
 		}
 
+		else if (t.cs() == "loadgame") {
+			p.skip_spaces();
+			string name = normalize_filename(p.verbatim_item());
+			string const path = getMasterFilePath();
+			// We want to preserve relative / absolute filenames,
+			// therefore path is only used for testing
+			if (!makeAbsPath(name, path).exists()) {
+				// The file extension is probably missing.
+				// Now try to find it out.
+				char const * const lyxskak_format[] = {"fen", 0};
+				string const lyxskak_name =
+					find_file(name, path, lyxskak_format);
+				if (!lyxskak_name.empty())
+					name = lyxskak_name;
+			}
+			if (makeAbsPath(name, path).exists())
+				fix_relative_filename(name);
+			else
+				cerr << "Warning: Could not find file '"
+				     << name << "'." << endl;
+			context.check_layout(os);
+			begin_inset(os, "External\n\ttemplate ");
+			os << "ChessDiagram\n\tfilename "
+			   << name << "\n";
+			end_inset(os);
+			context.check_layout(os);
+			// after a \loadgame follows a \showboard
+			if (p.get_token().asInput() == "showboard")
+				p.get_token();
+		}
+
 		else {
 			// try to see whether the string is in unicodesymbols
 			// Only use text mode commands, since we are in text mode here,

status.20x

@@ -28,6 +28,10 @@ What's new
 
 * TEX2LYX IMPROVEMENTS
 
+- Chess diagram and Spreadsheet external templates are imported
+
+- tabular* environments are imported
+
 
 * USER INTERFACE
 
@@ -61,6 +65,22 @@ What's new
 
 * TEX2LYX
 
+- tex2lyx roundtips pollutes preamble with color code (bug 7845).
+
+- tex2lyx support for \date{} (bug 7844).
+
+- Latex import whitespace (bug 7668).
+
+- asme2e issues (bug 6449).
+
+- tex2lyx: problem with macros nested in \foreignlanguage (bug 5187).
+
+- tex2lyx booktabs support (bug 4553).
+
+- tex2lyx change tracking support (bug 4213).
+
+- tex2lyx problems with character style switches (bug 3036).
+
 
 * USER INTERFACE