Assure correct spacing of colored items in mathed

When coloring a non-ordinary math atom, explicitly apply its
math class, so that to leave unchanged the spacing around it.
Fixes #11827.

src/mathed/InsetMath.h

@@ -72,6 +72,7 @@ class InsetMathArray;
 class InsetMathAMSArray;
 class InsetMathBrace;
 class InsetMathChar;
+class InsetMathClass;
 class InsetMathDelim;
 class InsetMathFracBase;
 class InsetMathFrac;
@@ -150,6 +151,7 @@ public:
 	virtual InsetMathBrace          * asBraceInset()          { return nullptr; }
 	virtual InsetMathBrace const    * asBraceInset() const    { return nullptr; }
 	virtual InsetMathChar const     * asCharInset() const     { return nullptr; }
+	virtual InsetMathClass const    * asClassInset() const    { return nullptr; }
 	virtual InsetMathDelim          * asDelimInset()          { return nullptr; }
 	virtual InsetMathDelim const    * asDelimInset() const    { return nullptr; }
 	virtual InsetMathFracBase       * asFracBaseInset()       { return nullptr; }

src/mathed/InsetMathClass.h

@@ -38,6 +38,10 @@ public:
 	void infoize(odocstream & os) const;
 	///
 	InsetCode lyxCode() const { return MATH_CLASS_CODE; }
+	///
+	InsetMathClass * asClassInset() { return this; }
+	///
+	InsetMathClass const * asClassInset() const { return this; }
 
 private:
 	virtual Inset * clone() const;

src/mathed/InsetMathColor.cpp

@@ -96,7 +96,46 @@ void InsetMathColor::validate(LaTeXFeatures & features) const
 
 void InsetMathColor::write(WriteStream & os) const
 {
-	if (normalcolor(color_))
+	// We have to ensure correct spacing when the front and/or back
+	// atoms are not ordinary ones (bug 11827).
+	docstring const frontclass =
+		cell(0).size() ? class_to_string(cell(0).front()->mathClass())
+		               : from_ascii("mathord");
+	docstring const backclass =
+		cell(0).size() ? class_to_string(cell(0).back()->mathClass())
+		               : from_ascii("mathord");
+	bool adjchk = os.latex() && !os.inMathClass() && (normalcolor(color_) || oldstyle_);
+	bool adjust_front = frontclass != "mathord" && adjchk;
+	bool adjust_back = backclass != "mathord" && adjchk;
+	docstring const colswitch =
+		oldstyle_ ? from_ascii("{\\color{") + color_ + from_ascii("}")
+			  : from_ascii("{\\normalcolor ");
+
+	if (adjust_front && adjust_back) {
+		os << '\\' << frontclass << colswitch << cell(0).front() << '}';
+		if (cell(0).size() > 2) {
+			os << colswitch;
+			for (size_t i = 1; i < cell(0).size() - 1; ++i)
+				os << cell(0)[i];
+			os << '}';
+		}
+		if (cell(0).size() > 1)
+			os << '\\' << backclass << colswitch << cell(0).back() << '}';
+	} else if (adjust_front) {
+		os << '\\' << frontclass << colswitch << cell(0).front() << '}';
+		if (cell(0).size() > 1) {
+			os << colswitch;
+			for (size_t i = 1; i < cell(0).size(); ++i)
+				os << cell(0)[i];
+			os << '}';
+		}
+	} else if (adjust_back) {
+		os << colswitch;
+		for (size_t i = 0; i < cell(0).size() - 1; ++i)
+			os << cell(0)[i];
+		os << '}' << '\\' << backclass << colswitch << cell(0).back()
+		   << '}';
+	} else if (normalcolor(color_))
 		// reset to default color inside another color inset
 		os << "{\\normalcolor " << cell(0) << '}';
 	else if (oldstyle_)

src/mathed/InsetMathNest.cpp

@@ -330,6 +330,7 @@ void InsetMathNest::write(WriteStream & os) const
 	ModeSpecifier specifier(os, currentMode(), lockedMode());
 	docstring const latex_name = name();
 	os << '\\' << latex_name;
+	os.inMathClass(asClassInset());
 	for (size_t i = 0; i < nargs(); ++i) {
 		Changer dummy = os.changeRowEntry(TexRow::mathEntry(id(),i));
 		os << '{' << cell(i) << '}';
@@ -340,6 +341,7 @@ void InsetMathNest::write(WriteStream & os) const
 		os << "\\lyxlock";
 		os.pendingSpace(true);
 	}
+	os.inMathClass(false);
 }
 
 

src/mathed/MathStream.cpp

@@ -131,7 +131,7 @@ WriteStream::WriteStream(otexrowstream & os, bool fragile, bool latex,
 	  output_(output), pendingspace_(false), pendingbrace_(false),
 	  textmode_(false), locked_(0), ascii_(0), canbreakline_(true),
 	  mathsout_(false), ulemcmd_(NONE), line_(0), encoding_(encoding),
-	  row_entry_(TexRow::row_none)
+	  row_entry_(TexRow::row_none), mathclass_(false)
 {}
 
 

src/mathed/MathStream.h

@@ -101,6 +101,10 @@ public:
 	void asciiOnly(bool ascii);
 	/// tell whether to use only ascii chars when producing latex code
 	bool asciiOnly() const { return ascii_; }
+	/// tell whether we are in a MathClass inset
+	void inMathClass(bool mathclass) { mathclass_ = mathclass; };
+	/// tell whether we are in a MathClass inset
+	bool inMathClass() const { return mathclass_; }
 	/// LaTeX encoding
 	Encoding const * encoding() const { return encoding_; }
 
@@ -142,6 +146,8 @@ private:
 	Encoding const * encoding_;
 	/// Row entry we are in
 	TexRow::RowEntry row_entry_;
+	/// whether we are in a MathClass inset
+	bool mathclass_;
 };
 
 ///