special character "allowbreak" (ZWSP), fileformat change

New special character to mark an optional line break
without inserting a hyphen (ZWSP). See #10585.

Corresponds to the Unicode character U+200B ZERO WIDTH SPACE

 This isn't a “space”. It is an invisible character that can be used
 to provide line break opportunities.
 ​http://unicode.org/notes/tn27/

While the literal Unicode character can be used in the LyX
file, it is invisible in the GUI.

For visible feedback, the patch adds a new special character "allowbreak".

The small mark is inspired by LibereOffice.
A tooltip is added.

development/FORMAT

@@ -6,6 +6,10 @@ changes happened in particular if possible. A good example would be
 2010-01-10 entry.
 
 -----------------------
+2017-04-19 Günter Milde <milde@lyx.org>
+ 	* Format incremented to 541: changes \SpecialChar:
+	  - new argument "allowbreak" to mark an optional line break
+	    without inserting a hyphen (= ZWSP). See #10585.
 
 2017-04-15 Uwe Stöhr <uwestoehr@web.de>
  	* Format incremented to 540: support for rotated float placements

lib/lyx2lyx/lyx_2_3.py

@@ -2122,6 +2122,24 @@ def revert_rotfloat(document):
 
     i = i + 1
 
+def convert_allowbreak(document):
+    " Zero widths Space-inset -> \SpecialChar allowbreak. "
+    body = "\n".join(document.body)
+    body = body.replace("\\begin_inset space \hspace{}\n"
+                        "\\length 0dd\n"
+                        "\\end_inset\n\n",
+                        "\\SpecialChar allowbreak\n")
+    document.body = body.split("\n")
+
+def revert_allowbreak(document):
+    " \SpecialChar allowbreak -> Zero widths Space-inset. "
+    body = "\n".join(document.body)
+    body = body.replace("\\SpecialChar allowbreak\n",
+                        "\\begin_inset space \hspace{}\n"
+                        "\\length 0dd\n"
+                        "\\end_inset\n\n")
+    document.body = body.split("\n")
+
 
 ##
 # Conversion hub
@@ -2160,10 +2178,12 @@ convert = [
            [537, []],
            [538, [convert_mathindent]],
            [539, []],
-           [540, []]
+           [540, []],
+           [541, [convert_allowbreak]],
            ]
 
 revert =  [
+           [540, [revert_allowbreak]],
            [539, [revert_rotfloat]],
            [538, [revert_baselineskip]],
            [537, [revert_mathindent]],

lib/ui/stdmenus.inc

@@ -433,6 +433,7 @@ Menuset
 		Separator
 		Item "Hyphenation Point|H" "specialchar-insert hyphenation"
 		Item "Ligature Break|k" "specialchar-insert ligature-break"
+		Item "Optional Line Break|B" "specialchar-insert allowbreak"
 		Item "Ragged Line Break|R" "newline-insert newline"
 		Item "Justified Line Break|J" "newline-insert linebreak"
 		Separator

src/LaTeXFeatures.cpp

@@ -79,6 +79,15 @@ static docstring const lyxarrow_def = from_ascii(
 	"{\\leavevmode\\,$\\triangleleft$\\,\\allowbreak}\n"
 	"{\\leavevmode\\,$\\triangleright$\\,\\allowbreak}}");
 
+// ZERO WIDTH SPACE (ZWSP) is actually not a space character
+// but marks a line break opportunity. Several commands provide a
+// line break opportunity. They differ in side-effects:
+// \allowbreak prevents hyphenation after hyphen or dash + ZWSP
+// \linebreak[<n>] takes an optional argument denoting "urgency".
+// The \LyXZeroWidthSpace wrapper allows customization in the preamble.
+static docstring const lyxZWSP_def = from_ascii(
+	"\\newcommand*\\LyXZeroWidthSpace{\\hspace{0pt}}");
+
 // for quotes without babel. This does not give perfect results, but
 // anybody serious about non-english quotes should use babel (JMarc).
 
@@ -1317,6 +1326,9 @@ TexString LaTeXFeatures::getMacros() const
 	if (mustProvide("lyxarrow"))
 		macros << lyxarrow_def << '\n';
 
+	if (mustProvide("lyxzerowidthspace"))
+		macros << lyxZWSP_def << '\n';
+
 	if (!usePolyglossia() && mustProvide("textgreek")) {
 	    // ensure LGR font encoding is defined also if fontenc is not loaded by LyX
 	   	if (params_.main_font_encoding() == "default")

src/LyXAction.cpp

@@ -3532,8 +3532,9 @@ void LyXAction::init()
  * \var lyx::FuncCode lyx::LFUN_SPECIALCHAR_INSERT
  * \li Action: Inserts various characters into the document.
  * \li Syntax: specialchar-insert <CHAR>
- * \li Params: <CHAR>: hyphenation, ligature-break, slash, nobreakdash, dots,
-                       end-of-sentence, menu-separator, lyx, tex, latex, latex2e.
+ * \li Params: <CHAR>: hyphenation, allowbreak, ligature-break, slash,
+   	   		   		   nobreakdash, dots, end-of-sentence, menu-separator, 
+ 					   lyx, tex, latex, latex2e.
  * \li Origin: JSpitzm, 6 Dec 2007
  * \endvar
  */

src/Text3.cpp

@@ -1243,6 +1243,8 @@ void Text::dispatch(Cursor & cur, FuncRequest & cmd)
 		string const name = to_utf8(cmd.argument());
 		if (name == "hyphenation")
 			specialChar(cur, InsetSpecialChar::HYPHENATION);
+		else if (name == "allowbreak")
+			specialChar(cur, InsetSpecialChar::ALLOWBREAK);
 		else if (name == "ligature-break")
 			specialChar(cur, InsetSpecialChar::LIGATURE_BREAK);
 		else if (name == "slash")

src/insets/InsetSpecialChar.cpp

@@ -86,6 +86,32 @@ int logoWidth(FontInfo const & font, InsetSpecialChar::Kind kind) {
 
 }
 
+docstring InsetSpecialChar::toolTip(BufferView const &, int, int) const
+{
+	docstring message;
+	switch (kind_) {
+		case ALLOWBREAK:
+			message = from_ascii("Optional Line Break (ZWSP)");
+			break;
+		case LIGATURE_BREAK:
+			message = from_ascii("Ligature Break (ZWNJ)");
+			break;
+		case END_OF_SENTENCE:
+			message = from_ascii("End of Sentence");
+			break;
+		case HYPHENATION:
+			message = from_ascii("Hyphenation Point");
+			break;
+		case SLASH:
+			message = from_ascii("Breakable Slash");
+			break;
+		case NOBREAKDASH:
+			message = from_ascii("Protected Hyphen (SHY)");
+			break;
+	}
+	return message;
+}
+
 
 void InsetSpecialChar::metrics(MetricsInfo & mi, Dimension & dim) const
 {
@@ -97,6 +123,9 @@ void InsetSpecialChar::metrics(MetricsInfo & mi, Dimension & dim) const
 
 	docstring s;
 	switch (kind_) {
+		case ALLOWBREAK:
+			dim.wid = fm.em() / 8;
+			break;
 		case LIGATURE_BREAK:
 			s = from_ascii("|");
 			break;
@@ -224,6 +253,18 @@ void InsetSpecialChar::draw(PainterInfo & pi, int x, int y) const
 		pi.pain.text(x, y, char_type('-'), font);
 		break;
 	}
+	case ALLOWBREAK:
+	{
+		// A small vertical line
+		int const asc = theFontMetrics(pi.base.font).ascent('x');
+		int const desc = theFontMetrics(pi.base.font).descent('g');
+		int const x0 = x; // x + 1; // FIXME: incline,
+		int const x1 = x; // x - 1; // similar to LibreOffice?
+		int const y0 = y + desc;
+		int const y1 = y - asc / 3;
+		pi.pain.line(x0, y1, x1, y0, Color_special);
+		break;
+	}
 	case LIGATURE_BREAK:
 	{
 		font.setColor(Color_special);
@@ -292,6 +333,9 @@ void InsetSpecialChar::write(ostream & os) const
 	case HYPHENATION:
 		command = "softhyphen";
 		break;
+	case ALLOWBREAK:
+		command = "allowbreak";
+		break;
 	case LIGATURE_BREAK:
 		command = "ligaturebreak";
 		break;
@@ -334,6 +378,8 @@ void InsetSpecialChar::read(Lexer & lex)
 
 	if (command == "softhyphen")
 		kind_ = HYPHENATION;
+	else if (command == "allowbreak")
+		kind_ = ALLOWBREAK;
 	else if (command == "ligaturebreak")
 		kind_ = LIGATURE_BREAK;
 	else if (command == "endofsentence")
@@ -366,6 +412,9 @@ void InsetSpecialChar::latex(otexstream & os,
 	case HYPHENATION:
 		os << "\\-";
 		break;
+	case ALLOWBREAK:
+		os << "\\LyXZeroWidthSpace" << termcmd;
+		break;
 	case LIGATURE_BREAK:
 		os << "\\textcompwordmark" << termcmd;
 		break;
@@ -420,6 +469,9 @@ int InsetSpecialChar::plaintext(odocstringstream & os,
 	switch (kind_) {
 	case HYPHENATION:
 		return 0;
+	case ALLOWBREAK:
+		os.put(0x200b);
+		return 1;
 	case LIGATURE_BREAK:
 		os.put(0x200c);
 		return 1;
@@ -460,6 +512,10 @@ int InsetSpecialChar::docbook(odocstream & os, OutputParams const &) const
 {
 	switch (kind_) {
 	case HYPHENATION:
+		break;
+	case ALLOWBREAK:
+		os.put(0x200b);
+		break;
 	case LIGATURE_BREAK:
 		break;
 	case END_OF_SENTENCE:
@@ -500,6 +556,9 @@ docstring InsetSpecialChar::xhtml(XHTMLStream & xs, OutputParams const &) const
 	switch (kind_) {
 	case HYPHENATION:
 		break;
+	case ALLOWBREAK:
+		xs << XHTMLStream::ESCAPE_NONE << "&#8203;";
+		break;
 	case LIGATURE_BREAK:
 		xs << XHTMLStream::ESCAPE_NONE << "&#8204;";
 		break;
@@ -538,8 +597,9 @@ docstring InsetSpecialChar::xhtml(XHTMLStream & xs, OutputParams const &) const
 void InsetSpecialChar::toString(odocstream & os) const
 {
 	switch (kind_) {
+	case ALLOWBREAK:
 	case LIGATURE_BREAK:
-		// Do not output ZERO WIDTH NON JOINER here
+		// Do not output ZERO WIDTH SPACE and ZERO WIDTH NON JOINER here
 		// Spell checker would choke on it.
 		return;
 	default:
@@ -562,6 +622,8 @@ void InsetSpecialChar::forOutliner(docstring & os, size_t const,
 
 void InsetSpecialChar::validate(LaTeXFeatures & features) const
 {
+	if (kind_ == ALLOWBREAK)
+		features.require("lyxzerowidthspace");
 	if (kind_ == MENU_SEPARATOR)
 		features.require("lyxarrow");
 	if (kind_ == NOBREAKDASH)
@@ -591,8 +653,8 @@ bool InsetSpecialChar::isLineSeparator() const
 	// Paragraph::stripLeadingSpaces nukes the characters which
 	// have this property. I leave the code here, since it should
 	// eventually be made to work. (JMarc 20020327)
-	return kind_ == HYPHENATION || kind_ == MENU_SEPARATOR
-		|| kind_ == SLASH;
+	return kind_ == HYPHENATION || kind_ == ALLOWBREAK
+	    || kind_ == MENU_SEPARATOR || kind_ == SLASH;
 #else
 	return false;
 #endif

src/insets/InsetSpecialChar.h

@@ -30,6 +30,8 @@ public:
 	enum Kind {
 		/// Optional hyphenation point (\-)
 		HYPHENATION,
+		/// Optional line wrap point without hyphen (ZWSP)
+		ALLOWBREAK,
 		/// Ligature break point (\textcompwordmark)
 		LIGATURE_BREAK,
 		/// ... (\ldots)
@@ -59,6 +61,8 @@ public:
 	///
 	Kind kind() const;
 	///
+	docstring toolTip(BufferView const & bv, int x, int y) const;
+	///
 	void metrics(MetricsInfo &, Dimension &) const;
 	///
 	void draw(PainterInfo & pi, int x, int y) const;

src/tex2lyx/TODO.txt

@@ -159,3 +159,12 @@ General
 
 * Use the language information provided by Language.cpp and the languages file (for babel/lyx/polyglossia name, quote style etc.)
   instead of hardcoding this information in Preamble.cpp.
+
+* Store alias commands for characters in a file with
+  "lib/unicodesymbols"-format, e.g.
+  
+    0x200b "\\dots" "" "" "" # HORIZONTAL ELLIPSIS
+    0x200b "\\ldots" "" "" "\\ldots" # HORIZONTAL ELLIPSIS
+    0x200b "\\textellipsis" "" "" "" # HORIZONTAL ELLIPSIS
+  
+  Look up LaTeX->Unicode conversion in the combined files.

src/tex2lyx/test/CJK.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/CJKutf8.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/DummyDocument.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/Dummy~Document.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/XeTeX-polyglossia.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/algo2e.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/box-color-size-space-align.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true
@@ -2876,7 +2876,7 @@ Characters
 \begin_layout Standard
 
 \size normal
-from package "ascii" and "ifsym": ®↨◻◼◙
+from package "ascii" and "ifsym": ®â†¨â—»â—¼â—™
 \end_layout
 
 \begin_layout Standard

src/tex2lyx/test/test-insets-basic.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-insets.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-memoir.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-modules.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-refstyle-theorems.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-scr.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-structure.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/verbatim.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 540
+\lyxformat 541
 \begin_document
 \begin_header
 \save_transient_properties true

src/version.h

@@ -32,8 +32,8 @@ extern char const * const lyx_version_info;
 
 // Do not remove the comment below, so we get merge conflict in
 // independent branches. Instead add your own.
-#define LYX_FORMAT_LYX 540 // uwestoehr: enable placement for rotated floats
-#define LYX_FORMAT_TEX2LYX 540
+#define LYX_FORMAT_LYX 541 // milde: \SpecialChar allowbreak
+#define LYX_FORMAT_TEX2LYX 541
 
 #if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX
 #ifndef _MSC_VER