Fix lyx2lyx math package defaults (bug #9069)

lyx2lyx sets all new math packages to off when converting from old formats.
This is correct if any command which would cause an automatic package loading
exists in the document. However, it is wrong if no command exists: This leads
to problems if later a command is added (bug #9069), which is especially
annoying for templates.
The fix consists of two parts:
1) convert_use_package() considers now the used commands like
   revert_use_package(), and uses them to decide whether to set the package to
   auto or off.
2) convert_undertilde() and revert_undertilde() use a slightly adjusted copy of
   convert_use_package() and revert_use_package(), so that the bug is also
   fixed for undertilde. We cannot use the latter functions directly, because
   of "\usepackage undertilde" vs. "\use_undertilde".
(cherry picked from commit b73a59932e)

Conflicts:
	status.21x

lib/lyx2lyx/lyx_2_1.py

@@ -253,6 +253,7 @@ def revert_visible_space(document):
       document.body[i:end + 1] = subst
 
 
+undertilde_commands = ["utilde"]
 def convert_undertilde(document):
     " Load undertilde automatically "
     i = find_token(document.header, "\\use_mathdots" , 0)
@@ -264,52 +265,60 @@ def convert_undertilde(document):
         document.warning("Malformed LyX document: Can't find \\use_mathdots.")
         return;
     j = find_token(document.preamble, "\\usepackage{undertilde}", 0)
-    if j == -1:
+    if j != -1:
-        document.header.insert(i + 1, "\\use_undertilde 0")
+        # package was loaded in the preamble, convert this to header setting for round trip
-    else:
+        document.header.insert(i + 1, "\\use_undertilde 2") # on
-        document.header.insert(i + 1, "\\use_undertilde 2")
         del document.preamble[j]
+    else:
+        j = 0
+        while True:
+            j = find_token(document.body, '\\begin_inset Formula', j)
+            if j == -1:
+                break
+            k = find_end_of_inset(document.body, j)
+            if k == -1:
+                document.warning("Malformed LyX document: Can't find end of Formula inset at line " + str(j))
+                j += 1
+                continue
+            code = "\n".join(document.body[j:k])
+            for c in undertilde_commands:
+                if code.find("\\%s" % c) != -1:
+                    # at least one of the commands was found - need to switch package off
+                    document.header.insert(i + 1, "\\use_undertilde 0") # off
+                    return
+            j = k
+        # no command was found - set to auto (bug 9069)
+        document.header.insert(i + 1, "\\use_undertilde 1") # auto
+
 
 
 def revert_undertilde(document):
     " Load undertilde if used in the document "
-    undertilde = find_token(document.header, "\\use_undertilde" , 0)
+    regexp = re.compile(r'(\\use_undertilde)')
-    if undertilde == -1:
+    i = find_re(document.header, regexp, 0)
-      document.warning("No \\use_undertilde line. Assuming auto.")
+    value = "1" # default is auto
-    else:
+    if i != -1:
-      val = get_value(document.header, "\\use_undertilde", undertilde)
+        value = get_value(document.header, "\\use_undertilde" , i).split()[0]
-      del document.header[undertilde]
+        del document.header[i]
-      try:
+    if value == "2": # on
-        usetilde = int(val)
-      except:
-        document.warning("Invalid \\use_undertilde value: " + val + ". Assuming auto.")
-        # probably usedots has not been changed, but be safe.
-        usetilde = 1
-
-      if usetilde == 0:
-        # do not load case
-        return
-      if usetilde == 2:
-        # force load case
         add_to_preamble(document, ["\\usepackage{undertilde}"])
-        return
+    elif value == "1": # auto
-
+        i = 0
-    # so we are in the auto case. we want to load undertilde if \utilde is used.
+        while True:
-    i = 0
+            i = find_token(document.body, '\\begin_inset Formula', i)
-    while True:
+            if i == -1:
-      i = find_token(document.body, '\\begin_inset Formula', i)
+                return
-      if i == -1:
+            j = find_end_of_inset(document.body, i)
-        return
+            if j == -1:
-      j = find_end_of_inset(document.body, i)
+                document.warning("Malformed LyX document: Can't find end of Formula inset at line " + str(i))
-      if j == -1:
+                i += 1
-        document.warning("Malformed LyX document: Can't find end of Formula inset at line " + str(i))
+                continue
-        i += 1
+            code = "\n".join(document.body[i:j])
-        continue
+            for c in undertilde_commands:
-      code = "\n".join(document.body[i:j])
+                if code.find("\\%s" % c) != -1:
-      if code.find("\\utilde") != -1:
+                    add_to_preamble(document, ["\\usepackage{undertilde}"])
-        add_to_preamble(document, ["\\@ifundefined{utilde}{\\usepackage{undertilde}}"])
+                    return
-        return
+            i = j
-      i = j
 
 
 def revert_negative_space(document):
@@ -569,17 +578,50 @@ def revert_use_packages(document):
         j += 1
 
 
-def convert_use_package(document, pkg):
+def convert_use_package(document, pkg, commands, oldauto):
+    # oldauto defines how the version we are converting from behaves:
+    # if it is true, the old version uses the package automatically.
+    # if it is false, the old version never uses the package.
     i = find_token(document.header, "\\use_package", 0)
     if i == -1:
         document.warning("Malformed LyX document: Can't find \\use_package.")
         return;
     j = find_token(document.preamble, "\\usepackage{" + pkg + "}", 0)
-    if j == -1:
+    if j != -1:
-        document.header.insert(i + 1, "\\use_package " + pkg + " 0")
+        # package was loaded in the preamble, convert this to header setting for round trip
-    else:
+        document.header.insert(i + 1, "\\use_package " + pkg + " 2") # on
-        document.header.insert(i + 1, "\\use_package " + pkg + " 2")
         del document.preamble[j]
+    # If oldauto is true we have two options:
+    # We can either set the package to auto - this is correct for files in
+    # format 425 to 463, and may create a conflict for older files which use
+    # any command in commands with a different definition.
+    # Or we can look whether any command in commands is used, and set it to
+    # auto if not and to off if yes. This will not create a conflict, but will
+    # create uncompilable documents for files in format 425 to 463, which use
+    # any command in commands.
+    # We choose the first option since its error is less likely.
+    elif oldauto:
+        document.header.insert(i + 1, "\\use_package " + pkg + " 1") # auto
+    else:
+        j = 0
+        while True:
+            j = find_token(document.body, '\\begin_inset Formula', j)
+            if j == -1:
+                break
+            k = find_end_of_inset(document.body, j)
+            if k == -1:
+                document.warning("Malformed LyX document: Can't find end of Formula inset at line " + str(j))
+                j += 1
+                continue
+            code = "\n".join(document.body[j:k])
+            for c in commands:
+                if code.find("\\%s" % c) != -1:
+                    # at least one of the commands was found - need to switch package off
+                    document.header.insert(i + 1, "\\use_package " + pkg + " 0") # off
+                    return
+            j = k
+        # no command was found - set to auto (bug 9069)
+        document.header.insert(i + 1, "\\use_package " + pkg + " 1") # auto
 
 
 def revert_use_package(document, pkg, commands, oldauto):
@@ -613,31 +655,24 @@ def revert_use_package(document, pkg, commands, oldauto):
             i = j
 
 
-def convert_use_mathtools(document):
+mathtools_commands = ["mathclap", "mathllap", "mathrlap", \
-    "insert use_package mathtools"
-    convert_use_package(document, "mathtools")
-
-
-def revert_use_mathtools(document):
-    "remove use_package mathtools"
-    commands = ["mathclap", "mathllap", "mathrlap", \
                 "lgathered", "rgathered", "vcentcolon", "dblcolon", \
                 "coloneqq", "Coloneqq", "coloneq", "Coloneq", "eqqcolon", \
                 "Eqqcolon", "eqcolon", "Eqcolon", "colonapprox", \
                 "Colonapprox", "colonsim", "Colonsim"]
-    revert_use_package(document, "mathtools", commands, False)
+def convert_use_mathtools(document):
+    "insert use_package mathtools"
+    convert_use_package(document, "mathtools", mathtools_commands, False)
 
 
-def convert_use_stmaryrd(document):
+def revert_use_mathtools(document):
-    "insert use_package stmaryrd"
+    "remove use_package mathtools"
-    convert_use_package(document, "stmaryrd")
+    revert_use_package(document, "mathtools", mathtools_commands, False)
 
 
-def revert_use_stmaryrd(document):
+# commands provided by stmaryrd.sty but LyX uses other packages:
-    "remove use_package stmaryrd"
+# boxdot lightning, bigtriangledown, bigtriangleup
-    # commands provided by stmaryrd.sty but LyX uses other packages:
+stmaryrd_commands = ["shortleftarrow", "shortrightarrow", "shortuparrow", \
-    # boxdot lightning, bigtriangledown, bigtriangleup
-    commands = ["shortleftarrow", "shortrightarrow", "shortuparrow", \
                     "shortdownarrow", "Yup", "Ydown", "Yleft", "Yright", \
                     "varcurlyvee", "varcurlywedge", "minuso", "baro", \
                     "sslash", "bbslash", "moo", "varotimes", "varoast", \
@@ -668,19 +703,25 @@ def revert_use_stmaryrd(document):
                     "varcopyright", "longarrownot", "Longarrownot", \
                     "Mapsto", "mapsfrom", "Mapsfrom" "Longmapsto", \
                     "longmapsfrom", "Longmapsfrom"]
-    revert_use_package(document, "stmaryrd", commands, False)
+def convert_use_stmaryrd(document):
+    "insert use_package stmaryrd"
+    convert_use_package(document, "stmaryrd", stmaryrd_commands, False)
 
 
+def revert_use_stmaryrd(document):
+    "remove use_package stmaryrd"
+    revert_use_package(document, "stmaryrd", stmaryrd_commands, False)
 
+
+stackrel_commands = ["stackrel"]
 def convert_use_stackrel(document):
     "insert use_package stackrel"
-    convert_use_package(document, "stackrel")
+    convert_use_package(document, "stackrel", stackrel_commands, False)
 
 
 def revert_use_stackrel(document):
     "remove use_package stackrel"
-    commands = ["stackrel"]
+    revert_use_package(document, "stackrel", stackrel_commands, False)
-    revert_use_package(document, "stackrel", commands, False)
 
 
 def convert_cite_engine_type(document):
@@ -739,11 +780,11 @@ def revert_cite_engine_type_default(document):
     document.header[i] = "\\cite_engine_type " + engine_type
 
 
+cancel_commands = ["cancel", "bcancel", "xcancel", "cancelto"]
 # this is the same, as revert_use_cancel() except for the default
 def revert_cancel(document):
     "add cancel to the preamble if necessary"
-    commands = ["cancelto", "cancel", "bcancel", "xcancel"]
+    revert_use_package(document, "cancel", cancel_commands, False)
-    revert_use_package(document, "cancel", commands, False)
 
 
 def revert_verbatim(document):
@@ -1047,13 +1088,12 @@ def revert_use_amssymb(document):
 
 def convert_use_cancel(document):
     "insert use_package cancel"
-    convert_use_package(document, "cancel")
+    convert_use_package(document, "cancel", cancel_commands, True)
 
 
 def revert_use_cancel(document):
     "remove use_package cancel"
-    commands = ["cancel", "bcancel", "xcancel", "cancelto"]
+    revert_use_package(document, "cancel", cancel_commands, True)
-    revert_use_package(document, "cancel", commands, True)
 
 
 def revert_ancientgreek(document):
@@ -4465,6 +4505,9 @@ convert = [
            [422, [convert_use_packages]],
            [423, [convert_use_mathtools]],
            [424, [convert_cite_engine_type]],
+           # No convert_cancel, since cancel will be loaded automatically
+           # in format 425 without any possibility to switch it off.
+           # This has been fixed in format 464.
            [425, []],
            [426, []],
            [427, []],