Fix bug 3293 by Bernhard Roider:

This changes the semantics of isOK() and operator(), comments from Bernhard below:

With the old version of lyxlex it was _impossible_ to check whether reading an integer, float, ... succeeded or not. The current solution to check for is.bad() in some cases and in other cases use is.good() does not give the desired information. Moreover the result of is.bad() depends on the stl implementation and behaves different for linux and windows.

the bug was introduced by the patch that fixed the bug that crashed lyx when "inset-insert ert" was executed from the command buffer.
The lexer has the method isOK() which reflects the status of the stream is.
The operators void* and ! are not really well defined (they depend on the value of is.bad()). What is missing is a test if the last reading operation was successful and thus the returned value is valid.
That's what i implemented in this patch.

The new rule for using the lexer:

if you want to know if the lexer still has data to read (either from the stream or from the pushed token) then use "lex.isOK()".
If you want to test if the last reading operation was successful then use eg. "if (lex) {...}" or unsuccessful then use eg. "if (!lex) {...}"

an example:

int readParam(LyxLex &lex) {

    int param = 1; // default value
    if (lex.isOK()) { // the lexer has data to read
        int p;    // temporary variable
        lex >> p;
        if (lex) param = p; // only use the input if the reading operation was successful
    }
    return param;
}




git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@17569 a592a061-630c-0410-9148-cb99ea01b6c8

src/ToolbarBackend.C

@@ -79,12 +79,12 @@ void ToolbarBackend::read(LyXLex & lex)
 	Toolbar tb;
 	tb.name = lex.getString();
 	lex.next(true);
-	if (!lex.isOK()) {
+	tb.gui_name = lex.getString();
+	if (!lex) {
 		lyxerr << "ToolbarBackend::read: Malformed toolbar "
 			"description " <<  lex.getString() << endl;
 		return;
 	}
-	tb.gui_name = lex.getString();
 
 	bool quit = false;
 

src/buffer.C

@@ -653,7 +653,7 @@ Buffer::ReadStatus Buffer::readFile(LyXLex & lex, FileName const & filename,
 	lex.next();
 	string const token(lex.getString());
 
-	if (!lex.isOK()) {
+	if (!lex) {
 		Alert::error(_("Document could not be read"),
 			     bformat(_("%1$s could not be read."), from_utf8(filename.absFilename())));
 		return failure;

src/dociterator.C

@@ -234,14 +234,14 @@ DocIterator::col_type DocIterator::col() const
 
 MathArray const & DocIterator::cell() const
 {
-	BOOST_ASSERT(inMathed());
+//	BOOST_ASSERT(inMathed());
 	return top().cell();
 }
 
 
 MathArray & DocIterator::cell()
 {
-	BOOST_ASSERT(inMathed());
+//	BOOST_ASSERT(inMathed());
 	return top().cell();
 }
 

src/frontends/controllers/ControlLog.C

@@ -43,7 +43,7 @@ bool ControlLog::initialiseParams(string const & data)
 
 	string logtype, logfile;
 	lex >> logtype;
-	if (lex.isOK()) {
+	if (lex) {
 		lex.next(true);
 		logfile = lex.getString();
 	}

src/frontends/controllers/ControlParagraph.C

@@ -53,7 +53,12 @@ bool ControlParagraph::initialiseParams(string const & data)
 		} else if (token == "update") {
 			lex.next();
 			bool const accept = lex.getBool();
-			action = accept ? 1 : 2;
+			if (lex) {
+				action = accept ? 1 : 2;
+			} else {
+				// Unrecognised update option
+				return false;
+			}
 		} else if (!token.empty()) {
 			// Unrecognised token
 			return false;

src/insets/insetbox.C

@@ -531,15 +531,17 @@ void InsetBoxParams::read(LyXLex & lex)
 	if (!lex.isOK())
 		return;
 
-	if (lex.isOK()) {
-		lex.next();
-		type = lex.getString();
-	}
-	if (!lex.isOK())
+	lex.next();
+	type = lex.getString();
+
+	if (!lex)
 		return;
+
 	lex.next();
 	string token;
 	token = lex.getString();
+	if (!lex)
+		return;
 	if (token == "position") {
 		lex.next();
 		// The [0] is needed. We need the first and only char in
@@ -549,10 +551,11 @@ void InsetBoxParams::read(LyXLex & lex)
 		lyxerr << "InsetBox::Read: Missing 'position'-tag!" << token << endl;
 		lex.pushToken(token);
 	}
-	if (!lex.isOK())
-		return;
+
 	lex.next();
 	token = lex.getString();
+	if (!lex)
+		return;
 	if (token == "hor_pos") {
 		lex.next();
 		hor_pos = lex.getString()[0];
@@ -560,10 +563,11 @@ void InsetBoxParams::read(LyXLex & lex)
 		lyxerr << "InsetBox::Read: Missing 'hor_pos'-tag!" << token << endl;
 		lex.pushToken(token);
 	}
-	if (!lex.isOK())
-		return;
+
 	lex.next();
 	token = lex.getString();
+	if (!lex)
+		return;
 	if (token == "has_inner_box") {
 		lex.next();
 		inner_box = lex.getInteger();
@@ -572,10 +576,10 @@ void InsetBoxParams::read(LyXLex & lex)
 		lex.pushToken(token);
 	}
 
-	if (!lex.isOK())
-		return;
 	lex.next();
 	token = lex.getString();
+	if (!lex)
+		return;
 	if (token == "inner_pos") {
 		lex.next();
 		inner_pos = lex.getString()[0];
@@ -584,10 +588,11 @@ void InsetBoxParams::read(LyXLex & lex)
 			<< token << endl;
 		lex.pushToken(token);
 	}
-	if (!lex.isOK())
-		return;
+
 	lex.next();
 	token = lex.getString();
+	if (!lex)
+		return;
 	if (token == "use_parbox") {
 		lex.next();
 		use_parbox = lex.getInteger();
@@ -595,10 +600,11 @@ void InsetBoxParams::read(LyXLex & lex)
 		lyxerr << "InsetBox::Read: Missing 'use_parbox'-tag!" << endl;
 		lex.pushToken(token);
 	}
-	if (!lex.isOK())
-		return;
+
 	lex.next();
 	token = lex.getString();
+	if (!lex)
+		return;
 	if (token == "width") {
 		lex.next();
 		width = LyXLength(lex.getString());
@@ -606,10 +612,11 @@ void InsetBoxParams::read(LyXLex & lex)
 		lyxerr << "InsetBox::Read: Missing 'width'-tag!" << endl;
 		lex.pushToken(token);
 	}
-	if (!lex.isOK())
-		return;
+
 	lex.next();
 	token = lex.getString();
+	if (!lex)
+		return;
 	if (token == "special") {
 		lex.next();
 		special = lex.getString();
@@ -617,10 +624,11 @@ void InsetBoxParams::read(LyXLex & lex)
 		lyxerr << "InsetBox::Read: Missing 'special'-tag!" << endl;
 		lex.pushToken(token);
 	}
-	if (!lex.isOK())
-		return;
+
 	lex.next();
 	token = lex.getString();
+	if (!lex)
+		return;
 	if (token == "height") {
 		lex.next();
 		height = LyXLength(lex.getString());
@@ -628,10 +636,11 @@ void InsetBoxParams::read(LyXLex & lex)
 		lyxerr << "InsetBox::Read: Missing 'height'-tag!" << endl;
 		lex.pushToken(token);
 	}
-	if (!lex.isOK())
-		return;
+
 	lex.next();
 	token = lex.getString();
+	if (!lex)
+		return;
 	if (token == "height_special") {
 		lex.next();
 		height_special = lex.getString();

src/insets/insetert.C

@@ -466,7 +466,7 @@ void InsetERTMailer::string2params(string const & in,
 
 	int s;
 	lex >> s;
-	if (lex.isOK())
+	if (lex)
 		status = static_cast<InsetCollapsable::CollapseStatus>(s);
 }
 

src/insets/insetnote.C

@@ -105,7 +105,7 @@ void InsetNoteParams::read(LyXLex & lex)
 {
 	string label;
 	lex >> label;
-	if (lex.isOK())
+	if (lex)
 		type = notetranslator().find(label);
 }
 

src/insets/insettabular.C

@@ -231,11 +231,11 @@ void InsetTabular::read(Buffer const & buf, LyXLex & lex)
 
 	lex.next();
 	string token = lex.getString();
-	while (lex.isOK() && (token != "\\end_inset")) {
+	while (lex && token != "\\end_inset") {
 		lex.next();
 		token = lex.getString();
 	}
-	if (token != "\\end_inset") {
+	if (!lex) {
 		lex.printError("Missing \\end_inset at this point. "
 			       "Read: `$$Token'");
 	}

src/insets/insetvspace.C

@@ -88,7 +88,7 @@ void InsetVSpace::read(Buffer const &, LyXLex & lex)
 	BOOST_ASSERT(lex.isOK());
 	string vsp;
 	lex >> vsp;
-	if (lex.isOK())
+	if (lex)
 		space_ = VSpace(vsp);
 
 	string end_token;
@@ -257,7 +257,7 @@ void InsetVSpaceMailer::string2params(string const & in, VSpace & vspace)
 
 	string vsp;
 	lex >> vsp;
-	if (lex.isOK())
+	if (lex)
 		vspace = VSpace(vsp);
 }
 

src/lyxlex.C

@@ -53,7 +53,7 @@ LyXLex::~LyXLex()
 
 bool LyXLex::isOK() const
 {
-	return pimpl_->is.good();
+	return pimpl_->inputAvailable();
 }
 
 
@@ -124,8 +124,16 @@ int LyXLex::lex()
 
 int LyXLex::getInteger() const
 {
+	lastReadOk_ = pimpl_->status == LEX_DATA || pimpl_->status == LEX_TOKEN;
+	if (!lastReadOk_) {
+		pimpl_->printError("integer token missing");
+		return -1;
+	}
+
 	if (isStrInt(pimpl_->getString()))
 		return convert<int>(pimpl_->getString());
+
+	lastReadOk_ = false;
 	pimpl_->printError("Bad integer `$$Token'");
 	return -1;
 }
@@ -136,9 +144,17 @@ double LyXLex::getFloat() const
 	// replace comma with dot in case the file was written with
 	// the wrong locale (should be rare, but is easy enough to
 	// avoid).
+	lastReadOk_ = pimpl_->status == LEX_DATA || pimpl_->status == LEX_TOKEN;
+	if (!lastReadOk_) {
+		pimpl_->printError("float token missing");
+		return -1;
+	}
+
 	string const str = subst(pimpl_->getString(), ",", ".");
 	if (isStrDbl(str))
 		return convert<double>(str);
+
+	lastReadOk_ = false;
 	pimpl_->printError("Bad float `$$Token'");
 	return -1;
 }
@@ -146,13 +162,23 @@ double LyXLex::getFloat() const
 
 string const LyXLex::getString() const
 {
+	lastReadOk_ = pimpl_->status == LEX_DATA || pimpl_->status == LEX_TOKEN;
+
+	if (lastReadOk_)
 	return pimpl_->getString();
+
+	return string();
 }
 
 
 docstring const LyXLex::getDocString() const
 {
-	return pimpl_->getDocString();
+	lastReadOk_ = pimpl_->status == LEX_DATA || pimpl_->status == LEX_TOKEN;
+	
+	if (lastReadOk_)
+		return pimpl_->getDocString();
+
+	return docstring();
 }
 
 
@@ -164,7 +190,7 @@ string const LyXLex::getLongString(string const & endtoken)
 	string str, prefix;
 	bool firstline = true;
 
-	while (isOK()) {
+	while (pimpl_->is) { //< eatLine only reads from is, not from pushTok
 		if (!eatLine())
 			// blank line in the file being read
 			continue;
@@ -197,7 +223,7 @@ string const LyXLex::getLongString(string const & endtoken)
 		str += ltrim(tmpstr, "\t") + '\n';
 	}
 
-	if (!isOK()) {
+	if (!pimpl_->is) {
 		printError("Long string not ended by `" + endtoken + '\'');
 	}
 
@@ -208,11 +234,14 @@ string const LyXLex::getLongString(string const & endtoken)
 bool LyXLex::getBool() const
 {
 	if (pimpl_->getString() == "true") {
+		lastReadOk_ = true;
 		return true;
 	} else if (pimpl_->getString() != "false") {
 		pimpl_->printError("Bad boolean `$$Token'. "
 				   "Use \"false\" or \"true\"");
+		lastReadOk_ = false;
 	}
+	lastReadOk_ = true;
 	return false;
 }
 
@@ -246,13 +275,13 @@ LyXLex::operator void const *() const
 	// use fail() here. However, our implementation of getString() et al.
 	// can cause the eof() and fail() bits to be set, even though we
 	// haven't tried to read 'em.
-	return pimpl_->is.bad() ? 0 : this;
+	return lastReadOk_? this: 0;
 }
 
 
 bool LyXLex::operator!() const
 {
-	return pimpl_->is.bad();
+	return !lastReadOk_;
 }
 
 
@@ -261,6 +290,8 @@ LyXLex & LyXLex::operator>>(std::string & s)
 	if (isOK()) {
 		next();
 		s = getString();
+	} else {
+		lastReadOk_ = false;
 	}
 	return *this;
 }
@@ -271,6 +302,8 @@ LyXLex & LyXLex::operator>>(docstring & s)
 	if (isOK()) {
 		next();
 		s = getDocString();
+	} else {
+		lastReadOk_ = false;
 	}
 	return *this;
 }
@@ -281,6 +314,8 @@ LyXLex & LyXLex::operator>>(double & s)
 	if (isOK()) {
 		next();
 		s = getFloat();
+	} else {
+		lastReadOk_ = false;
 	}
 	return *this;
 }
@@ -291,6 +326,8 @@ LyXLex & LyXLex::operator>>(int & s)
 	if (isOK()) {
 		next();
 		s = getInteger();
+	} else {
+		lastReadOk_ = false;
 	}
 	return *this;
 }
@@ -301,6 +338,8 @@ LyXLex & LyXLex::operator>>(unsigned int & s)
 	if (isOK()) {
 		next();
 		s = getInteger();
+	} else {
+		lastReadOk_ = false;
 	}
 	return *this;
 }
@@ -311,6 +350,8 @@ LyXLex & LyXLex::operator>>(bool & s)
 	if (isOK()) {
 		next();
 		s = getBool();
+	} else {
+		lastReadOk_ = false;
 	}
 	return *this;
 }

src/lyxlex.h

@@ -37,8 +37,22 @@ struct keyword_item {
 };
 
 /** Generalized simple lexical analizer.
-    It can be used for simple syntax parsers, like lyxrc,
-    texclass and others to come.
+	Use the method isOK() to check if there is still data available 
+	for lexing. Use one of the the operators void* or ! to test if
+	the last reading operation was successful.
+	
+	Example:
+
+	int readParam(LyxLex &lex) {
+		int param = 1; // default value
+		if (lex.isOK()) { // the lexer has data to read
+			int p;    // temporary variable
+			lex >> p;
+			if (lex) param = p; // only use the input if reading was successful
+		}
+		return param;
+	} 
+
     @see lyxrc.C for an example of usage.
   */
 class LyXLex : boost::noncopyable {
@@ -61,14 +75,16 @@ public:
 	};
 
 	/// stream is open and end of stream is not reached
-	/// FIXME: Rename to good() since this is the name of the
-	/// corresponding std::stream method.
+	/// FIXME: test also if pushTok is not empty
+	/// FIXME: the method should be renamed to something like 
+	///        dataAvailable(), in order to reflect the real behavior
 	bool isOK() const;
-	/// stream is ok
-	/// FIXME: This does not behave like the std::stream counterpart.
+	/// FIXME: The next two operators should be replaced by one method
+	///        called e.g. lastReadOk(), in order to reflect the real 
+	///        behavior
+	/// last read operation was successful.
 	operator void const *() const;
-	/// stream is not ok
-	/// FIXME: This does not behave like the std::stream counterpart.
+	/// last read operation was not successful
 	bool operator!() const;
 	/// return true if able to open file, else false
 	bool setFile(support::FileName const & filename);
@@ -84,7 +100,7 @@ public:
 	/// returns a lex code
 	int lex();
 
-	/** Just read athe next word. If esc is true remember that
+	/** Just read the next word. If esc is true remember that
 	    some chars might be escaped: "\ atleast
 	*/
 	bool next(bool esc = false);
@@ -162,6 +178,8 @@ private:
 	class Pimpl;
 	///
 	Pimpl * pimpl_;
+	///
+	mutable bool lastReadOk_;
 };
 
 

src/lyxlex_pimpl.C

@@ -204,15 +204,15 @@ bool LyXLex::Pimpl::next(bool esc /* = false */)
 		// There can have been a whole line pushed so
 		// we extract the first word and leaves the rest
 		// in pushTok. (Lgb)
-		if (pushTok.find(' ') != string::npos && pushTok[0] == '\\') {
+		if (pushTok[0] == '\\' && pushTok.find(' ') != string::npos) {
 			buff.clear();
 			pushTok = split(pushTok, buff, ' ');
-			return true;
 		} else {
 			buff = pushTok;
 			pushTok.clear();
-			return true;
 		}
+		status = LEX_TOKEN;
+		return true;
 	}
 	if (!esc) {
 		unsigned char c = 0; // getc() returns an int
@@ -306,26 +306,8 @@ bool LyXLex::Pimpl::next(bool esc /* = false */)
 			c = cc;
 
 			// skip ','s
-			if (c == ',') continue;
-
-			if (c == '\\') {
-				// escape
-				buff.clear();
-
-				do {
-					if (c == '\\') {
-						// escape the next char
-						is.get(cc);
-						c = cc;
-					}
-					buff.push_back(c);
-					is.get(cc);
-					c = cc;
-				} while (c > ' ' && c != ',' && is);
-
-				status = LEX_TOKEN;
+			if (c == ',') 
 				continue;
-			}
 
 			if (c == commentChar) {
 				// Read rest of line (fast :-)
@@ -457,6 +439,9 @@ bool LyXLex::Pimpl::eatLine()
 		buff.resize(buff.size() - 1);
 		status = LEX_DATA;
 		return true;
+	} else if (buff.length() > 0) { // last line
+		status = LEX_DATA;
+		return true;
 	} else {
 		return false;
 	}
@@ -469,15 +454,15 @@ bool LyXLex::Pimpl::nextToken()
 		// There can have been a whole line pushed so
 		// we extract the first word and leaves the rest
 		// in pushTok. (Lgb)
-		if (pushTok.find(' ') != string::npos && pushTok[0] == '\\') {
+		if (pushTok[0] == '\\' && pushTok.find(' ') != string::npos) {
 			buff.clear();
 			pushTok = split(pushTok, buff, ' ');
-			return true;
 		} else {
 			buff = pushTok;
 			pushTok.clear();
-			return true;
 		}
+		status = LEX_TOKEN;
+		return true;
 	}
 
 	status = 0;
@@ -519,6 +504,11 @@ bool LyXLex::Pimpl::nextToken()
 }
 
 
+bool LyXLex::Pimpl::inputAvailable() {
+	return is.good(); 
+}
+
+
 void LyXLex::Pimpl::pushToken(string const & pt)
 {
 	pushTok = pt;

src/lyxlex_pimpl.h

@@ -65,6 +65,8 @@ public:
 	bool eatLine();
 	///
 	bool nextToken();
+	/// test if there is a pushed token or the stream is ok
+	bool inputAvailable();
 	///
 	void pushToken(std::string const &);
 	/// fb_ is only used to open files, the stream is accessed through is.