Fix several false errors "stray '}' in text" that were caused by

misinterpreting "\}" as "}" when it occured inside a pair of unescaped
braces, like in "\code{@\{*\}r||p\{1in\}@\{*\}}".
The reason for this bug is that Token::character() behaves differently in
tex2lyx than in mathed. See the comment in Parser.h for a more general fix.
For now I played on the safe side and only changed those places where I
definitely know that the old code was wrong.


git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@37117 a592a061-630c-0410-9148-cb99ea01b6c8

src/tex2lyx/Parser.cpp

@@ -128,6 +128,30 @@ bool Token::isAlnumASCII() const
 }
 
 
+#ifdef FILEDEBUG
+void debugToken(std::ostream & os, Token const & t, unsigned int flags)
+{
+	char sep = ' ';
+	os << "t: " << t << " flags: " << flags;
+	if (flags & FLAG_BRACE_LAST) { os << sep << "BRACE_LAST"; sep = '|'; }
+	if (flags & FLAG_RIGHT     ) { os << sep << "RIGHT"     ; sep = '|'; }
+	if (flags & FLAG_END       ) { os << sep << "END"       ; sep = '|'; }
+	if (flags & FLAG_BRACK_LAST) { os << sep << "BRACK_LAST"; sep = '|'; }
+	if (flags & FLAG_TEXTMODE  ) { os << sep << "TEXTMODE"  ; sep = '|'; }
+	if (flags & FLAG_ITEM      ) { os << sep << "ITEM"      ; sep = '|'; }
+	if (flags & FLAG_LEAVE     ) { os << sep << "LEAVE"     ; sep = '|'; }
+	if (flags & FLAG_SIMPLE    ) { os << sep << "SIMPLE"    ; sep = '|'; }
+	if (flags & FLAG_EQUATION  ) { os << sep << "EQUATION"  ; sep = '|'; }
+	if (flags & FLAG_SIMPLE2   ) { os << sep << "SIMPLE2"   ; sep = '|'; }
+	if (flags & FLAG_OPTION    ) { os << sep << "OPTION"    ; sep = '|'; }
+	if (flags & FLAG_BRACED    ) { os << sep << "BRACED"    ; sep = '|'; }
+	if (flags & FLAG_CELL      ) { os << sep << "CELL"      ; sep = '|'; }
+	if (flags & FLAG_TABBING   ) { os << sep << "TABBING"   ; sep = '|'; }
+	os << "\n";
+}
+#endif
+
+
 //
 // Parser
 //

src/tex2lyx/Parser.h

@@ -81,7 +81,17 @@ public:
 	std::string const & cs() const { return cs_; }
 	/// Returns the catcode of the token
 	CatCode cat() const { return cat_; }
-	///
+	/** Get the character of tokens that were constructed from a single
+	 * character input or a two character input and cat_ == catEscape.
+	 * FIXME: The intended usage is not clear. The Token class in
+	 *        ../mathed/MathParser.cpp (which is the anchestor of this
+	 *        class) uses a separate char member for this method. I
+	 *        believe that the intended usage is to not cover tokens with
+	 *        catEscape, e.g. \code
+	 *        return (cs_.empty() || cat_ == catEscape) ? 0 : cs_[0];
+	 *        \endcode
+	 *        All usages of this method should be checked. gb 2011-01-05
+	 */
 	char character() const { return cs_.empty() ? 0 : cs_[0]; }
 	/// Returns the token verbatim
 	std::string asInput() const;

src/tex2lyx/table.cpp

@@ -520,7 +520,7 @@ void parse_table(Parser & p, ostream & os, bool is_long_tabular,
 		Token const & t = p.get_token();
 
 #ifdef FILEDEBUG
-		cerr << "t: " << t << " flags: " << flags << "\n";
+		debugToken(cerr, t, flags);
 #endif
 
 		// comments and whitespace in hlines

src/tex2lyx/test/test-insets.tex

@@ -16,9 +16,11 @@
 % the following is useful when we have the old nomencl.sty package
 \providecommand{\printnomenclature}{\printglossary}
 \providecommand{\makenomenclature}{\makeglossary}
+\makenomenclature
 \usepackage{varioref}
 \usepackage{prettyref}
 \usepackage{makeidx}
+\makeindex
 
 \usepackage{graphicx}
 \usepackage{longtable}
@@ -167,7 +169,7 @@ at the end& of the table\\
 \endlastfoot
 \env{longtable}  columns  are specified& in the \\
 same way as  in the \env{tabular}& environment.\\
-%\code{@\{*\}r||p\{1in\}@\{*\}}& in this case.\\
+\code{@\{*\}r||p\{1in\}@\{*\}}& in this case.\\
 Each row ends with a& \code{\textbackslash\textbackslash} command.\\
 The \code{\textbackslash\textbackslash} command  has an& optional\\
 argument, just as in& the\\

src/tex2lyx/text.cpp

@@ -495,7 +495,8 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
 	unsigned int optargs = 0;
 	while (optargs < context.layout->optargs) {
 		eat_whitespace(p, os, context, false);
-		if (p.next_token().character() != '[') 
+		if (p.next_token().cat() == catEscape ||
+		    p.next_token().character() != '[') 
 			break;
 		p.get_token(); // eat '['
 		begin_inset(os, "OptArg\n");
@@ -505,14 +506,10 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
 		eat_whitespace(p, os, context, false);
 		++optargs;
 	}
-#if 0
-	// This is the code needed to parse required arguments, but 
-	// required arguments come into being only much later than the
-	// file format tex2lyx is presently outputting.
 	unsigned int reqargs = 0;
-	while (reqargs < context.layout->reqargs) {
+	while (LYX_FORMAT >= 392 && reqargs < context.layout->reqargs) {
 		eat_whitespace(p, os, context, false);
-		if (p.next_token().character() != '{') 
+		if (p.next_token().cat() != catBegin)
 			break;
 		p.get_token(); // eat '{'
 		begin_inset(os, "OptArg\n");
@@ -522,7 +519,6 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
 		eat_whitespace(p, os, context, false);
 		++reqargs;
 	}
-#endif
 	parse_text(p, os, FLAG_ITEM, outer, context);
 	context.check_end_layout(os);
 	if (parent_context.deeper_paragraph) {
@@ -1337,7 +1333,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 		Token const & t = p.get_token();
 
 #ifdef FILEDEBUG
-		cerr << "t: " << t << " flags: " << flags << "\n";
+		debugToken(cerr, t, flags);
 #endif
 
 		if (flags & FLAG_ITEM) {
@@ -1356,9 +1352,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			flags |= FLAG_LEAVE;
 		}
 
-		if (t.character() == ']' && (flags & FLAG_BRACK_LAST))
+		if (t.cat() != catEscape && t.character() == ']' &&
+		    (flags & FLAG_BRACK_LAST))
 			return;
-		if (t.character() == '}' && (flags & FLAG_BRACE_LAST))
+		if (t.cat() == catEnd && (flags & FLAG_BRACE_LAST))
 			return;
 
 		// If there is anything between \end{env} and \begin{env} we
@@ -1656,7 +1653,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			p.skip_spaces();
 			string s;
 			bool optarg = false;
-			if (p.next_token().character() == '[') {
+			if (p.next_token().cat() != catEscape &&
+			    p.next_token().character() == '[') {
 				p.get_token(); // eat '['
 				s = parse_text_snippet(p, FLAG_BRACK_LAST,
 						       outer, context);
@@ -1782,7 +1780,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 			os << "\\begin_layout " 
 			   << to_utf8(context.textclass.defaultLayout().name()) 
 			   << '\n';
-			if (p.next_token().character() == '[') {
+			if (p.next_token().cat() != catEscape &&
+			    p.next_token().character() == '[') {
 				p.get_token(); // eat '['
 				begin_inset(os, "OptArg\n");
 				os << "status collapsed\n";