Fix some lyx2lyx round-trip tests.

Fix cases where lyx2lyx adds changes with every round cycle
uncovered by recent ctest change.

Add optional `delete` argument to parser_tools.get*value():
If True, delete the matching line.

More efficient "allowbreak" con/reversion.

lib/lyx2lyx/lyx_2_0.py

@@ -22,7 +22,8 @@ import re, string
 import unicodedata
 import sys, os
 
-from parser_tools import find_token, find_end_of, find_tokens, \
+from parser_tools import del_complete_lines, \
+  find_token, find_end_of, find_tokens, \
   find_token_exact, find_end_of_inset, find_end_of_layout, \
   find_token_backwards, is_in_inset, get_value, get_quoted_value, \
   del_token, check_token, get_option_value
@@ -484,6 +485,15 @@ def revert_printindexall(document):
             document.body[i:k + 1] = subst
         i = i + 1
 
+strikeout_preamble = ['%  for proper underlining',
+                      r'\PassOptionsToPackage{normalem}{ulem}',
+                      r'\usepackage{ulem}']
+
+def convert_strikeout(document):
+    " Remove preamble code loading 'ulem' package. "
+    del_complete_lines(document.preamble,
+                       ['% Added by lyx2lyx']+strikeout_preamble)
+
 
 def revert_strikeout(document):
   " Reverts \\strikeout font attribute "
@@ -491,25 +501,32 @@ def revert_strikeout(document):
   changed = revert_font_attrs(document.body, "\\uwave", "\\uwave") or changed
   changed = revert_font_attrs(document.body, "\\strikeout", "\\sout")  or changed
   if changed == True:
-    insert_to_preamble(document, \
+    insert_to_preamble(document, strikeout_preamble)
-        ['%  for proper underlining',
-        '\\PassOptionsToPackage{normalem}{ulem}',
-        '\\usepackage{ulem}'])
 
 
+ulinelatex_preamble = ['% fix underbar in citations',
+    r'\let\cite@rig\cite',
+    r'\newcommand{\b@xcite}[2][\%]{\def\def@pt{\%}\def\pas@pt{#1}',
+    r'  \mbox{\ifx\def@pt\pas@pt\cite@rig{#2}\else\cite@rig[#1]{#2}\fi}}',
+    r'\renewcommand{\underbar}[1]{{\let\cite\b@xcite\uline{#1}}}']
+
+def convert_ulinelatex(document):
+    " Remove preamble code for \\uline font attribute. "
+    del_complete_lines(document.preamble,
+                       ['% Added by lyx2lyx']+ulinelatex_preamble)
+    for line in document.preamble:
+        print line
+
 def revert_ulinelatex(document):
-    " Reverts \\uline font attribute "
+    " Add preamble code for \\uline font attribute in citations. "
     i = find_token(document.body, '\\bar under', 0)
     if i == -1:
         return
-    insert_to_preamble(document,\
+    try:
-            ['%  for proper underlining',
+        document.preamble.index(r'\usepackage{ulem}')
-            '\\PassOptionsToPackage{normalem}{ulem}',
+    except ValueError:
-            '\\usepackage{ulem}',
+        insert_to_preamble(document, strikeout_preamble)
-            '\\let\\cite@rig\\cite',
+    insert_to_preamble(document, ulinelatex_preamble)
-            '\\newcommand{\\b@xcite}[2][\\%]{\\def\\def@pt{\\%}\\def\\pas@pt{#1}',
-            '  \\mbox{\\ifx\\def@pt\\pas@pt\\cite@rig{#2}\\else\\cite@rig[#1]{#2}\\fi}}',
-            '\\renewcommand{\\underbar}[1]{{\\let\\cite\\b@xcite\\uline{#1}}}'])
 
 
 def revert_custom_processors(document):
@@ -2468,9 +2485,9 @@ convert = [[346, []],
            [352, [convert_splitindex]],
            [353, []],
            [354, []],
-           [355, []],
+           [355, [convert_strikeout]],
            [356, []],
-           [357, []],
+           [357, [convert_ulinelatex]],
            [358, []],
            [359, [convert_nomencl_width]],
            [360, []],

lib/lyx2lyx/lyx_2_1.py

@@ -24,7 +24,8 @@ import sys, os
 
 # Uncomment only what you need to import, please.
 
-from parser_tools import count_pars_in_inset, del_token, find_token, find_token_exact, \
+from parser_tools import count_pars_in_inset, del_complete_lines, del_token, \
+    find_token, find_token_exact, \
     find_token_backwards, find_end_of, find_end_of_inset, find_end_of_layout, \
     find_end_of_sequence, find_re, get_option_value, get_containing_layout, \
     get_containing_inset, get_value, get_quoted_value, set_option_value
@@ -618,15 +619,16 @@ def convert_use_package(document, pkg, commands, oldauto):
     # oldauto defines how the version we are converting from behaves:
     # if it is true, the old version uses the package automatically.
     # if it is false, the old version never uses the package.
-    i = find_token(document.header, "\\use_package", 0)
+    i = find_token(document.header, "\\use_package")
     if i == -1:
         document.warning("Malformed LyX document: Can't find \\use_package.")
         return;
-    j = find_token(document.preamble, "\\usepackage{" + pkg + "}", 0)
+    packageline = "\\usepackage{%s}" % pkg
-    if j != -1:
+    if (del_complete_lines(document.preamble,
-        # package was loaded in the preamble, convert this to header setting for round trip
+                           ['% Added by lyx2lyx', packageline]) or
+        del_complete_lines(document.preamble, [packageline])):
+        # package was loaded in the preamble, convert this to header setting
         document.header.insert(i + 1, "\\use_package " + pkg + " 2") # on
-        del document.preamble[j]
     # If oldauto is true we have two options:
     # We can either set the package to auto - this is correct for files in
     # format 425 to 463, and may create a conflict for older files which use

lib/lyx2lyx/lyx_2_3.py

@@ -24,11 +24,11 @@ import sys, os
 
 # Uncomment only what you need to import, please.
 
-from parser_tools import del_token, del_value, del_complete_lines, \
+from parser_tools import (del_token, del_value, del_complete_lines,
-    find_end_of, find_end_of_layout, find_end_of_inset, find_re, \
+    find_complete_lines, find_end_of, find_end_of_layout, find_end_of_inset,
-    find_token, find_token_backwards, get_containing_layout, \
+    find_re, find_token, find_token_backwards,
-    get_bool_value, get_value, get_quoted_value
+    get_containing_layout, get_bool_value, get_value, get_quoted_value)
-#  find_tokens, find_token_exact, is_in_inset, \
+#  find_tokens, find_token_exact, is_in_inset,
 #  check_token, get_option_value
 
 from lyx2lyx_tools import add_to_preamble, put_cmd_in_ert, revert_font_attrs, \
@@ -1902,6 +1902,7 @@ def convert_dashligatures(document):
         document.header.insert(i, "\\use_dash_ligatures %s"
                                % str(use_dash_ligatures).lower())
 
+
 def revert_dashligatures(document):
     """Remove font ligature settings for en- and em-dashes.
     Revert conversion of \twodashes or \threedashes to literal dashes."""
@@ -1973,51 +1974,41 @@ def revert_xout(document):
 
 
 def convert_mathindent(document):
-    " add the \\is_math_indent tag "
+    """Add the \\is_math_indent tag.
+    """
+    k = find_token(document.header, "\\quotes_style") # where to insert
     # check if the document uses the class option "fleqn"
-    k = find_token(document.header, "\\quotes_style", 0)
+    options = get_value(document.header, "\\options")
-    regexp = re.compile(r'^.*fleqn.*')
+    if 'fleqn' in options:
-    i = find_re(document.header, regexp, 0)
-    if i != -1:
         document.header.insert(k, "\\is_math_indent 1")
-        # delete the found option
+        # delete the fleqn option
-        document.header[i] = document.header[i].replace(",fleqn", "")
+        i = find_token(document.header, "\\options")
-        document.header[i] = document.header[i].replace(", fleqn", "")
+        options = [option for option in options.split(",")
-        document.header[i] = document.header[i].replace("fleqn,", "")
+                   if option.strip() != "fleqn"]
-        j = find_re(document.header, regexp, 0)
+        if options:
-        if i == j:
+            document.header[i] = "\\options " + ",".join(options)
-            # then we have fleqn as the only option
+        else:
             del document.header[i]
     else:
         document.header.insert(k, "\\is_math_indent 0")
 
-
 def revert_mathindent(document):
     " Define mathindent if set in the document "
-    # first output the length
+    # emulate and delete \math_indentation
-    regexp = re.compile(r'(\\math_indentation)')
+    value = get_value(document.header, "\\math_indentation",
-    i = find_re(document.header, regexp, 0)
+                      default="default", delete=True)
+    if value != "default":
+        add_to_preamble(document, [r"\setlength{\mathindent}{%s}"%value])
+    # delete \is_math_indent and emulate via document class option
+    if not get_bool_value(document.header, "\\is_math_indent", delete=True):
+        return
+    i = find_token(document.header, "\\options")
     if i != -1:
-        value = get_value(document.header, "\\math_indentation" , i).split()[0]
+        document.header[i] = document.header[i].replace("\\options ",
-        if value != "default":
+                                                        "\\options fleqn,")
-            add_to_preamble(document, ["\\setlength{\\mathindent}{" + value + '}'])
-        del document.header[i]
-    # now set the document class option
-    regexp = re.compile(r'(\\is_math_indent 1)')
-    i = find_re(document.header, regexp, 0)
-    if i == -1:
-        regexp = re.compile(r'(\\is_math_indent)')
-        j = find_re(document.header, regexp, 0)
-        del document.header[j]
     else:
-        k = find_token(document.header, "\\options", 0)
+        l = find_token(document.header, "\\use_default_options")
-        if k != -1:
+        document.header.insert(l, "\\options fleqn")
-            document.header[k] = document.header[k].replace("\\options", "\\options fleqn,")
-            del document.header[i]
-        else:
-            l = find_token(document.header, "\\use_default_options", 0)
-            document.header.insert(l, "\\options fleqn")
-            del document.header[i + 1]
 
 
 def revert_baselineskip(document):
@@ -2126,24 +2117,31 @@ def revert_rotfloat(document):
     i = i + 1
 
 
+allowbreak_emulation =  [r"\begin_inset space \hspace{}",
+                         r"\length 0dd",
+                         r"\end_inset",
+                         r""]
+
 def convert_allowbreak(document):
     " Zero widths Space-inset -> \SpecialChar allowbreak. "
-    body = "\n".join(document.body)
+    lines = document.body
-    body = body.replace("\\begin_inset space \hspace{}\n"
+    i = find_complete_lines(lines, allowbreak_emulation, 2)
-                        "\\length 0dd\n"
+    while i != -1:
-                        "\\end_inset\n\n",
+        lines[i-1:i+4] = [lines[i-1] + r"\SpecialChar allowbreak"]
-                        "\\SpecialChar allowbreak\n")
+        i = find_complete_lines(lines, allowbreak_emulation, i)
-    document.body = body.split("\n")
 
 
 def revert_allowbreak(document):
     " \SpecialChar allowbreak -> Zero widths Space-inset. "
-    body = "\n".join(document.body)
+    i = 1
-    body = body.replace("\\SpecialChar allowbreak\n",
+    lines = document.body
-                        "\n\\begin_inset space \hspace{}\n"
+    while i < len(lines):
-                        "\\length 0dd\n"
+        if lines[i].endswith(r"\SpecialChar allowbreak"):
-                        "\\end_inset\n\n")
+            lines[i:i+1] = [lines[i].replace(r"\SpecialChar allowbreak", "")
-    document.body = body.split("\n")
+                           ] + allowbreak_emulation
+            i += 5
+        else:
+            i += 1
 
 
 def convert_mathnumberpos(document):
@@ -2227,7 +2225,7 @@ def revert_mathnumberingname(document):
         document.header[i] = "\\math_number_before 0"
         k = find_token(document.header, "\\options", 0)
         if k != -1:
-    	    document.header[k] = document.header[k].replace("\\options", "\\options reqno,")
+            document.header[k] = document.header[k].replace("\\options", "\\options reqno,")
         else:
             l = find_token(document.header, "\\use_default_options", 0)
             document.header.insert(l, "\\options reqno")
@@ -2240,7 +2238,8 @@ def revert_mathnumberingname(document):
 
 def convert_minted(document):
     " add the \\use_minted tag "
-    document.header.insert(-1, "\\use_minted 0")
+    i = find_token(document.header, "\\index ")
+    document.header.insert(i, "\\use_minted 0")
 
 
 def revert_minted(document):

lib/lyx2lyx/parser_tools.py

@@ -53,7 +53,7 @@ find_re(lines, rexp, start[, end]):
   As find_token, but rexp is a regular expression object,
   so it has to be passed as e.g.: re.compile(r'...').
 
-get_value(lines, token, start[, end[, default]):
+get_value(lines, token[, start[, end[, default[, delete]]]]):
   Similar to find_token, but it returns what follows the
   token on the found line. Example:
     get_value(document.header, "\\use_xetex", 0)
@@ -64,7 +64,7 @@ get_value(lines, token, start[, end[, default]):
   and is what is returned if we do not find anything. So you
   can use that to set a default.
 
-get_quoted_value(lines, token, start[, end[, default]]):
+get_quoted_value(lines, token[, start[, end[, default[, delete]]]]):
   Similar to get_value, but it will strip quotes off the
   value, if they are present. So use this one for cases
   where the value is normally quoted.
@@ -74,7 +74,7 @@ get_option_value(line, option):
       option="value"
   and returns value. Returns "" if not found.
 
-get_bool_value(lines, token, start[, end[, default]]):
+get_bool_value(lines, token[, start[, end[, default, delete]]]]):
   Like get_value, but returns a boolean.
 
 del_token(lines, token, start[, end]):
@@ -357,12 +357,15 @@ def find_across_lines(lines, sub, start=0, end=0):
     return -1
 
 
-def get_value(lines, token, start=0, end=0, default=""):
+def get_value(lines, token, start=0, end=0, default="", delete=False):
-    """ get_value(lines, token, start[[, end], default]) -> string
+    """Find `token` in `lines` and return part of line that follows it.
 
     Find the next line that looks like:
       token followed by other stuff
-    Returns "followed by other stuff" with leading and trailing
+
+    If `delete` is True, delete the line (if found).
+
+    Return "followed by other stuff" with leading and trailing
     whitespace removed.
     """
     i = find_token_exact(lines, token, start, end)
@@ -372,12 +375,14 @@ def get_value(lines, token, start=0, end=0, default=""):
     #  return lines.pop(i)[len(token):].strip() # or default
     # see test_parser_tools.py
     l = lines[i].split(None, 1)
+    if delete:
+        del(lines[i])
     if len(l) > 1:
         return l[1].strip()
     return default
 
 
-def get_quoted_value(lines, token, start=0, end=0, default=""):
+def get_quoted_value(lines, token, start=0, end=0, default="", delete=False):
     """ get_quoted_value(lines, token, start[[, end], default]) -> string
 
     Find the next line that looks like:
@@ -388,13 +393,13 @@ def get_quoted_value(lines, token, start=0, end=0, default=""):
     if they are there.
     Note that we will NOT strip quotes from default!
     """
-    val = get_value(lines, token, start, end, "")
+    val = get_value(lines, token, start, end, "", delete)
     if not val:
       return default
     return val.strip('"')
 
 
-def get_bool_value(lines, token, start=0, end=0, default=None):
+def get_bool_value(lines, token, start=0, end=0, default=None, delete=False):
     """ get_bool_value(lines, token, start[[, end], default]) -> string
 
     Find the next line that looks like:
@@ -404,7 +409,7 @@ def get_bool_value(lines, token, start=0, end=0, default=None):
     False if bool_value is 0 or false
     """
 
-    val = get_quoted_value(lines, token, start, end, "")
+    val = get_quoted_value(lines, token, start, end, default, delete)
 
     if val == "1" or val == "true":
         return True