Fix output of en- and em-dashes with TeX fonts

This commit fixes the regression introduced in 2.2 about the
output of en- and em-dashes. In 2.2 en- and em-dashes are output as
the \textendash and \textemdash macros when using TeX fonts, causing
changed output in old documents and also bugs (for example, #10490).

Now documents produced with older versions work again as intended,
while documents produced with 2.2 can be made to produce the exact
same output by simply checking "Don't use ligatures for en-and
em-dashes" in Document->Settings->Fonts.

When exporting documents using TeX fonts to earlier versions, in order
to avoid changed output, a zero-width space character is inserted after
each en/em-dash if dash ligatures are allowed. These characters are
removed when reloading  documents with 2.3, so that they don't accumulate.

development/FORMAT

@@ -7,6 +7,13 @@ changes happened in particular if possible. A good example would be
 
 -----------------------
 
+2017-03-19 Enrico Forestieri <forenr@lyx.org>
+	* Format incremented to 535: support for en/em-dash as ligatures.
+	  The en- and em-dashes (U+2013 and U+2014) are now exported as
+	  the font ligatures -- and --- when they would have been exported
+	  as the macros \textendash and \textemdash, unless instructed
+	  otherwise by a document preference.
+
 2017-02-04 Jürgen Spitzmüller <spitz@lyx.org>
 	* Format incremented to 534: Support for chapterbib
           - New buffer param value \multibib child

lib/RELEASE-NOTES

@@ -13,6 +13,12 @@
   be safely dissolved, as it will be automatically inserted at export time
   if needed, as usual.
 
+* LyX now outputs en- and em-dashes as -- and --- ligatures when exporting to
+  latex using TeX fonts, as done in version 2.1 and earlier. In version 2.2
+  they were instead output as the macros \textendash and \textemdash, causing
+  changed output with old documents and bugs. The 2.2 behavior can be restored
+  by don't allowing using dash ligatures in Document→Settings→Fonts.
+
 
 !!!The following pref variables were added in 2.3:
 
@@ -82,3 +88,16 @@
 
 !!Caveats when upgrading from earlier versions to 2.3.x
 
+* When loading documents created with LyX 2.2, you might need to check
+  "Don't use ligatures for en- and em-dashes" in Document→Settings→Fonts
+  to avoid changed output if they contain en- or em-dashes and use TeX fonts.
+  You don't need to do this for documents created with earlier versions.
+
+* If the "Use non-TeX fonts" and "Don't use ligatures for en- and em-dashes"
+  document preferences are not checked, when exporting documents containing
+  en- and em-dashes to the format of LyX 2.0 or earlier, the following line
+  has to be manually added to the unicodesymbols file of that LyX version:<br>
+  0x200b "\\hspace{0pt}" "" "" "" "" # ZERO WIDTH SPACE<br>
+  This avoids "uncodable character" issues if the document is actually
+  loaded by that LyX version. LyX 2.1 and later versions already have the
+  necessary definition in their unicodesymbols file.

lib/lyx2lyx/lyx_2_3.py

@@ -1842,6 +1842,97 @@ def revert_chapterbib(document):
     add_to_preamble(document, ["\\usepackage{chapterbib}"])
 
 
+def convert_dashligatures(document):
+    " Remove a zero-length space (U+200B) after en- and em-dashes. "
+
+    i = 0
+    while i < len(document.body):
+        words = document.body[i].split()
+        # Skip some document parts where dashes are not converted
+        if len(words) > 1 and words[0] == "\\begin_inset" and \
+           words[1] in ["CommandInset", "ERT", "External", "Formula", \
+                        "FormulaMacro", "Graphics", "IPA", "listings"]:
+            j = find_end_of_inset(document.body, i)
+            if j == -1:
+                document.warning("Malformed LyX document: Can't find end of " \
+                                 + words[1] + " inset at line " + str(i))
+                i += 1
+            else:
+                i = j
+            continue
+        if len(words) > 0 and words[0] in ["\\leftindent", \
+                "\\paragraph_spacing", "\\align", "\\labelwidthstring"]:
+            i += 1
+            continue
+
+        start = 0
+        while True:
+            j = document.body[i].find(u"\u2013", start) # en-dash
+            k = document.body[i].find(u"\u2014", start) # em-dash
+            if j == -1 and k == -1:
+                break
+            if j == -1 or (k != -1 and k < j):
+                j = k
+            after = document.body[i][j+1:]
+            if after.startswith(u"\u200B"):
+                document.body[i] = document.body[i][:j+1] + after[1:]
+            else:
+                if len(after) == 0 and document.body[i+1].startswith(u"\u200B"):
+                    document.body[i+1] = document.body[i+1][1:]
+                    break
+            start = j+1
+        i += 1
+
+
+def revert_dashligatures(document):
+    " Remove font ligature settings for en- and em-dashes. "
+    i = find_token(document.header, "\\use_dash_ligatures", 0)
+    if i == -1:
+        return
+    use_dash_ligatures = get_bool_value(document.header, "\\use_dash_ligatures", i)
+    del document.header[i]
+    use_non_tex_fonts = False
+    i = find_token(document.header, "\\use_non_tex_fonts", 0)
+    if i != -1:
+        use_non_tex_fonts = get_bool_value(document.header, "\\use_non_tex_fonts", i)
+    if not use_dash_ligatures or use_non_tex_fonts:
+        return
+
+    # Add a zero-length space (U+200B) after en- and em-dashes
+    i = 0
+    while i < len(document.body):
+        words = document.body[i].split()
+        # Skip some document parts where dashes are not converted
+        if len(words) > 1 and words[0] == "\\begin_inset" and \
+           words[1] in ["CommandInset", "ERT", "External", "Formula", \
+                        "FormulaMacro", "Graphics", "IPA", "listings"]:
+            j = find_end_of_inset(document.body, i)
+            if j == -1:
+                document.warning("Malformed LyX document: Can't find end of " \
+                                 + words[1] + " inset at line " + str(i))
+                i += 1
+            else:
+                i = j
+            continue
+        if len(words) > 0 and words[0] in ["\\leftindent", \
+                "\\paragraph_spacing", "\\align", "\\labelwidthstring"]:
+            i += 1
+            continue
+
+        start = 0
+        while True:
+            j = document.body[i].find(u"\u2013", start) # en-dash
+            k = document.body[i].find(u"\u2014", start) # em-dash
+            if j == -1 and k == -1:
+                break
+            if j == -1 or (k != -1 and k < j):
+                j = k
+            after = document.body[i][j+1:]
+            document.body[i] = document.body[i][:j+1] + u"\u200B" + after
+            start = j+1
+        i += 1
+    
+
 ##
 # Conversion hub
 #
@@ -1873,10 +1964,12 @@ convert = [
            [531, []],
            [532, [convert_literalparam]],
            [533, []],
-           [534, []]
+           [534, []],
+           [535, [convert_dashligatures]]
           ]
 
 revert =  [
+           [534, [revert_dashligatures]],
            [533, [revert_chapterbib]],
            [532, [revert_multibib]],
            [531, [revert_literalparam]],

src/BufferParams.cpp

@@ -415,6 +415,7 @@ BufferParams::BufferParams()
 	fonts_default_family = "default";
 	useNonTeXFonts = false;
 	use_microtype = false;
+	use_dash_ligatures = true;
 	fonts_expert_sc = false;
 	fonts_old_figures = false;
 	fonts_sans_scale[0] = 100;
@@ -812,6 +813,8 @@ string BufferParams::readToken(Lexer & lex, string const & token,
 		lex >> fonts_cjk;
 	} else if (token == "\\use_microtype") {
 		lex >> use_microtype;
+	} else if (token == "\\use_dash_ligatures") {
+		lex >> use_dash_ligatures;
 	} else if (token == "\\paragraph_separation") {
 		string parsep;
 		lex >> parsep;
@@ -1196,6 +1199,7 @@ void BufferParams::writeFile(ostream & os, Buffer const * buf) const
 		os << "\\font_cjk " << fonts_cjk << '\n';
 	}
 	os << "\\use_microtype " << convert<string>(use_microtype) << '\n';
+	os << "\\use_dash_ligatures " << convert<string>(use_dash_ligatures) << '\n';
 	os << "\\graphics " << graphics_driver << '\n';
 	os << "\\default_output_format " << default_output_format << '\n';
 	os << "\\output_sync " << output_sync << '\n';

src/BufferParams.h

@@ -280,6 +280,8 @@ public:
 	std::string fonts_cjk;
 	/// use LaTeX microtype package
 	bool use_microtype;
+	/// use font ligatures for en- and em-dashes
+	bool use_dash_ligatures;
 	///
 	Spacing & spacing();
 	Spacing const & spacing() const;

src/Paragraph.cpp

@@ -1274,6 +1274,21 @@ void Paragraph::Private::latexSpecialChar(otexstream & os,
 		// written. (Asger)
 		break;
 
+	case 0x2013:
+	case 0x2014:
+		if (bparams.use_dash_ligatures && !bparams.useNonTeXFonts) {
+			if (c == 0x2013) {
+				// en-dash
+				os << "--";
+				column +=2;
+			} else {
+				// em-dash
+				os << "---";
+				column +=3;
+			}
+			break;
+		}
+		// fall through
 	default:
 		if (c == '\0')
 			return;

src/frontends/qt4/GuiDocument.cpp

@@ -838,6 +838,8 @@ GuiDocument::GuiDocument(GuiView & lv)
 		this, SLOT(change_adaptor()));
 	connect(fontModule->microtypeCB, SIGNAL(clicked()),
 		this, SLOT(change_adaptor()));
+	connect(fontModule->dashesCB, SIGNAL(clicked()),
+		this, SLOT(change_adaptor()));
 	connect(fontModule->scaleSansSB, SIGNAL(valueChanged(int)),
 		this, SLOT(change_adaptor()));
 	connect(fontModule->scaleTypewriterSB, SIGNAL(valueChanged(int)),
@@ -3046,6 +3048,7 @@ void GuiDocument::applyView()
 		fromqstr(fontModule->cjkFontLE->text());
 
 	bp_.use_microtype = fontModule->microtypeCB->isChecked();
+	bp_.use_dash_ligatures = !fontModule->dashesCB->isChecked();
 
 	bp_.fonts_sans_scale[nontexfonts] = fontModule->scaleSansSB->value();
 	bp_.fonts_sans_scale[!nontexfonts] = fontModule->font_sf_scale;
@@ -3550,6 +3553,7 @@ void GuiDocument::paramsToDialog()
 		fontModule->cjkFontLE->setText(QString());
 	
 	fontModule->microtypeCB->setChecked(bp_.use_microtype);
+	fontModule->dashesCB->setChecked(!bp_.use_dash_ligatures);
 
 	fontModule->fontScCB->setChecked(bp_.fonts_expert_sc);
 	fontModule->fontOsfCB->setChecked(bp_.fonts_old_figures);

src/frontends/qt4/ui/FontUi.ui

@@ -242,7 +242,27 @@
      </property>
     </widget>
    </item>
+   <item row="10" column="1">
+    <widget class="QCheckBox" name="microtypeCB">
+     <property name="toolTip">
+      <string>Activate extensions such as character protrusion and font expansion via the microtype package</string>
+     </property>
+     <property name="text">
+      <string>Enable micr&amp;o-typographic extensions</string>
+     </property>
+    </widget>
+   </item>
    <item row="11" column="1">
+    <widget class="QCheckBox" name="dashesCB">
+     <property name="toolTip">
+      <string>Use \textendash and \textemdash instead of -- and --- for en- and em-dashes</string>
+     </property>
+     <property name="text">
+      <string>Don't use ligatures for en- and &amp;em-dashes</string>
+     </property>
+    </widget>
+   </item>
+   <item row="12" column="1">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -255,16 +275,6 @@
      </property>
     </spacer>
    </item>
-   <item row="10" column="1">
-    <widget class="QCheckBox" name="microtypeCB">
-     <property name="toolTip">
-      <string>Activate extensions such as character protrusion and font expansion via the microtype package</string>
-     </property>
-     <property name="text">
-      <string>Enable micr&amp;o-typographic extensions</string>
-     </property>
-    </widget>
-   </item>
   </layout>
  </widget>
  <layoutdefault spacing="6" margin="11"/>

src/tex2lyx/test/CJK.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/CJKutf8.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/DummyDocument.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/Dummy~Document.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/XeTeX-polyglossia.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/algo2e.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/box-color-size-space-align.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-insets-basic.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-insets.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-memoir.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-modules.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-refstyle-theorems.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-scr.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test-structure.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/test.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/tex2lyx/test/verbatim.lyx.lyx

@@ -1,5 +1,5 @@
 #LyX file created by tex2lyx 2.3
-\lyxformat 534
+\lyxformat 535
 \begin_document
 \begin_header
 \save_transient_properties true

src/version.h

@@ -32,8 +32,8 @@ extern char const * const lyx_version_info;
 
 // Do not remove the comment below, so we get merge conflict in
 // independent branches. Instead add your own.
-#define LYX_FORMAT_LYX 534 // spitz: chapterbib support
-#define LYX_FORMAT_TEX2LYX 534
+#define LYX_FORMAT_LYX 535 // ef: support for en/em-dash as ligatures
+#define LYX_FORMAT_TEX2LYX 535
 
 #if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX
 #ifndef _MSC_VER