Improve quotation mark opening/closing guess

Fixes: #8831

This introduces
* a new inset member isPartOfTextSequence() that returns
whether the inset produces something visible at the current position
in the text flow
* an isOpenPunctuation() test that returns whether a character is in the
'Punctuation, Open' unicode class. This is used instead of just checking
for two (Western, ASCII) opening brackets

It also fixes the isChar() and isLetter() value of InsetSpecialChar,
since some types have not been assigned correctly.

src/Text3.cpp

@@ -1548,11 +1548,40 @@ void Text::dispatch(Cursor & cur, FuncRequest & cmd)
 		while (pos > 0 && par.isDeleted(pos - 1))
 			--pos;
 
+		bool const inner = (cmd.getArg(0) == "single" || cmd.getArg(0) == "inner");
+
+		// Guess quote side.
+		// A space triggers an opening quote. This is passed if the preceding
+		// char/inset is a space or at paragraph start.
 		char_type c = ' ';
-		if (pos > 0 && (!cur.prevInset() || !cur.prevInset()->isSpace()))
-			c = par.getChar(pos - 1);
-		InsetQuotesParams::QuoteLevel const quote_level =
-				(cmd.getArg(0) == "single" || cmd.getArg(0) == "inner")
+		if (pos > 0 && !par.isSpace(pos - 1)) {
+			if (cur.prevInset() && cur.prevInset()->lyxCode() == QUOTE_CODE) {
+				// If an opening double quotation mark precedes, and this
+				// is a single quote, make it opening as well
+				InsetQuotes & ins =
+					static_cast<InsetQuotes &>(*cur.prevInset());
+				string const type = ins.getType();
+				if (!suffixIs(type, "ld") || !inner)
+					c = par.getChar(pos - 1);
+			}
+			else if (!cur.prevInset()
+			    || (cur.prevInset() && cur.prevInset()->isChar()))
+				// If a char precedes, pass that and let InsetQuote decide
+				c = par.getChar(pos - 1);
+			else {
+				while (pos > 0) {
+					if (par.getInset(pos - 1)
+					    && !par.getInset(pos - 1)->isPartOfTextSequence()) {
+						// skip "invisible" insets
+						--pos;
+						continue;
+					}
+					c = par.getChar(pos - 1);
+					break;
+				}
+			}
+		}
+		InsetQuotesParams::QuoteLevel const quote_level = inner
 				? InsetQuotesParams::SecondaryQuotes : InsetQuotesParams::PrimaryQuotes;
 		cur.insert(new InsetQuotes(cur.buffer(), c, quote_level, cmd.getArg(1), cmd.getArg(2)));
 		cur.buffer()->updateBuffer();

src/insets/Inset.h

@@ -411,6 +411,8 @@ public:
 
 	/// Is the content of this inset part of the output document?
 	virtual bool producesOutput() const { return true; }
+	/// Is the content of this inset part of the immediate (visible) text sequence?
+	virtual bool isPartOfTextSequence() const { return producesOutput(); }
 
 	/// \return Tool tip for this inset.
 	/// This default implementation returns an empty string. This can be

src/insets/InsetArgument.h

@@ -115,6 +115,8 @@ protected:
 	void doDispatch(Cursor & cur, FuncRequest & cmd);
 	///
 	Inset * clone() const { return new InsetArgument(*this); }
+	/// Is the content of this inset part of the immediate (visible) text sequence?
+	bool isPartOfTextSequence() const { return false; }
 	//@}
 };
 

src/insets/InsetBibitem.h

@@ -86,6 +86,8 @@ private:
 	void doDispatch(Cursor & cur, FuncRequest & cmd);
 	///
 	Inset * clone() const { return new InsetBibitem(*this); }
+	/// Is the content of this inset part of the immediate (visible) text sequence?
+	bool isPartOfTextSequence() const { return false; }
 	///@}
 
 	/// \name Private functions inherited from InsetCommand class

src/insets/InsetFloat.h

@@ -109,6 +109,8 @@ private:
 	void doDispatch(Cursor & cur, FuncRequest & cmd);
 	///
 	Inset * clone() const { return new InsetFloat(*this); }
+	/// Is the content of this inset part of the immediate (visible) text sequence?
+	bool isPartOfTextSequence() const { return false; }
 	///
 	TexString getCaption(OutputParams const &) const;
 

src/insets/InsetIndex.h

@@ -83,6 +83,8 @@ private:
 	std::string contextMenuName() const;
 	///
 	Inset * clone() const { return new InsetIndex(*this); }
+	/// Is the content of this inset part of the immediate text sequence?
+	bool isPartOfTextSequence() const { return false; }
 
 	///
 	friend class InsetIndexParams;

src/insets/InsetLabel.h

@@ -59,6 +59,8 @@ public:
 	///
 	void addToToc(DocIterator const & di, bool output_active,
 				  UpdateType utype) const;
+	/// Is the content of this inset part of the immediate (visible) text sequence?
+	bool isPartOfTextSequence() const { return false; }
 	//@}
 
 	/// \name Static public methods obligated for InsetCommand derived classes

src/insets/InsetMarginal.h

@@ -38,6 +38,8 @@ public:
 	///
 	void addToToc(DocIterator const & di, bool output_active,
 				  UpdateType utype) const;
+	/// Is the content of this inset part of the immediate (visible) text sequence?
+	bool isPartOfTextSequence() const { return false; }
 private:
 	///
 	Inset * clone() const { return new InsetMarginal(*this); }

src/insets/InsetQuotes.cpp

@@ -41,6 +41,7 @@
 #include "support/docstream.h"
 #include "support/gettext.h"
 #include "support/lstrings.h"
+#include "support/textutils.h"
 
 #include <string.h>
 
@@ -526,15 +527,10 @@ docstring InsetQuotes::layoutName() const
 void InsetQuotes::setSide(char_type c)
 {
 	// Decide whether opening or closing quote
-	switch (c) {
-	case ' ':
-	case '(':
-	case '[':
+	if (lyx::isSpace(c) || isOpenPunctuation(c))
 		side_ = InsetQuotesParams::OpeningQuote;// opening quote
-		break;
-	default:
+	else
 		side_ = InsetQuotesParams::ClosingQuote;// closing quote
-	}
 }
 
 

src/insets/InsetSpecialChar.cpp

@@ -571,6 +571,12 @@ void InsetSpecialChar::validate(LaTeXFeatures & features) const
 }
 
 
+bool InsetSpecialChar::isChar() const
+{
+	return kind_ != HYPHENATION || kind_ != LIGATURE_BREAK;
+}
+
+
 bool InsetSpecialChar::isLetter() const
 {
 	return kind_ == HYPHENATION || kind_ == LIGATURE_BREAK

src/insets/InsetSpecialChar.h

@@ -89,11 +89,13 @@ public:
 	void validate(LaTeXFeatures &) const;
 
 	/// should this inset be handled like a normal character?
-	bool isChar() const { return true; }
+	bool isChar() const;
 	/// is this equivalent to a letter?
 	bool isLetter() const;
 	/// should we break lines after this inset?
 	bool isLineSeparator() const;
+	/// Is the content of this inset part of the immediate (visible) text sequence?
+	bool isPartOfTextSequence() const { return isChar(); }
 private:
 	Inset * clone() const { return new InsetSpecialChar(*this); }
 

src/insets/InsetWrap.h

@@ -89,6 +89,8 @@ private:
 	docstring layoutName() const;
 	///
 	Inset * clone() const { return new InsetWrap(*this); }
+	/// Is the content of this inset part of the immediate (visible) text sequence?
+	bool isPartOfTextSequence() const { return false; }
 
 	///
 	InsetWrapParams params_;

src/support/lstrings.cpp

@@ -182,6 +182,18 @@ bool isASCII(char_type c)
 }
 
 
+bool isOpenPunctuation(char_type c)
+{
+	if (!is_utf16(c)) {
+		// assume that no non-utf16 character is an op
+		// c outside the UCS4 range is catched as well
+		return false;
+	}
+	QChar const qc = ucs4_to_qchar(c);
+	return qc.category() == QChar::Punctuation_Open;
+}
+
+
 namespace support {
 
 int compare_no_case(docstring const & s, docstring const & s2)

src/support/textutils.h

@@ -53,6 +53,9 @@ bool isAlnumASCII(char_type c);
 /// return whether \p c is in the ASCII range
 bool isASCII(char_type c);
 
+/// return whether \p c is in the 'Punctuation, Open' unicode category
+bool isOpenPunctuation(char_type c);
+
 } // namespace lyx
 
 #endif // TEXTUTILS_H