From c6610453e0845383da8949492cdc142e985afb46 Mon Sep 17 00:00:00 2001 From: Georg Baum Date: Thu, 16 Jun 2016 22:05:56 +0200 Subject: [PATCH] Use unicode cmdline args consistently Previously the commandline arguments were processed in an operating system dependent encoding if running under python 2. Now they are converted to unicode during the parsing, using the more modern argpase instead of optparse. The individual conversion methods do no longer need to know anything about commandline encoding. This fixes a bug similar to #10218 if running under python 2 which I probably introduced during the python 3 conversion. --- lib/lyx2lyx/LyX.py | 18 +++++++------- lib/lyx2lyx/lyx2lyx | 54 ++++++++++++++++++++++++------------------ lib/lyx2lyx/lyx_1_4.py | 4 ++-- lib/lyx2lyx/lyx_2_2.py | 26 ++++++++------------ 4 files changed, 52 insertions(+), 50 deletions(-) diff --git a/lib/lyx2lyx/LyX.py b/lib/lyx2lyx/LyX.py index 6cbc6ffffa..3ee9a68baf 100644 --- a/lib/lyx2lyx/LyX.py +++ b/lib/lyx2lyx/LyX.py @@ -204,10 +204,10 @@ def get_encoding(language, inputencoding, format, cjk_encoding): class LyX_base: """This class carries all the information of the LyX file.""" - def __init__(self, end_format = 0, input = "", output = "", error = "", - debug = default_debug__, try_hard = 0, cjk_encoding = '', - final_version = "", systemlyxdir = '', language = "english", - encoding = "auto"): + def __init__(self, end_format = 0, input = u'', output = u'', error = u'', + debug = default_debug__, try_hard = 0, cjk_encoding = u'', + final_version = u'', systemlyxdir = u'', language = u'english', + encoding = u'auto'): """Arguments: end_format: final format that the file should be converted. (integer) @@ -459,7 +459,7 @@ class LyX_base: # Since we do not know the encoding yet we need to read the input as # bytes in binary mode, and convert later to unicode. - if input and input != '-': + if input and input != u'-': self.dir = os.path.dirname(os.path.abspath(input)) try: gzip.open(input).readline() @@ -469,7 +469,7 @@ class LyX_base: self.input = open(input, 'rb') self.compressed = False else: - self.dir = '' + self.dir = u'' self.input = os.fdopen(sys.stdin.fileno(), 'rb') self.compressed = False @@ -817,9 +817,9 @@ class LyX_base: class File(LyX_base): " This class reads existing LyX files." - def __init__(self, end_format = 0, input = "", output = "", error = "", - debug = default_debug__, try_hard = 0, cjk_encoding = '', - final_version = '', systemlyxdir = ''): + def __init__(self, end_format = 0, input = u'', output = u'', error = u'', + debug = default_debug__, try_hard = 0, cjk_encoding = u'', + final_version = u'', systemlyxdir = u''): LyX_base.__init__(self, end_format, input, output, error, debug, try_hard, cjk_encoding, final_version, systemlyxdir) diff --git a/lib/lyx2lyx/lyx2lyx b/lib/lyx2lyx/lyx2lyx index 04b23028b3..9d038305e1 100755 --- a/lib/lyx2lyx/lyx2lyx +++ b/lib/lyx2lyx/lyx2lyx @@ -19,58 +19,66 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA " Program used to convert between different versions of the lyx file format." -import optparse +import argparse import sys import LyX +# Provide support for both python 2 and 3 +PY2 = sys.version_info[0] == 2 +if PY2: + # argparse returns strings in the commandline encoding, we need to convert. + # sys.getdefaultencoding() would not always be correct, see + # http://legacy.python.org/dev/peps/pep-0383/ + def cmd_arg(arg): + return arg.decode(sys.getfilesystemencoding()) +else: + cmd_arg = str +# End of code to support for both python 2 and 3 + def main(): args = {} - args["usage"] = "usage: %prog [options] [file]" - - args["version"] = """lyx2lyx, version %s -Copyright (C) 2011 The LyX Team, José Matos and Dekel Tsur""" % LyX.version__ + args["usage"] = "%(prog)s [options] [file]" args["description"] = """Convert old lyx file to newer format, files can be compressed with gzip. If there no file is specified then the standard input is assumed, in this case gziped files are not handled.""" - parser = optparse.OptionParser(**args) + parser = argparse.ArgumentParser(**args) parser.set_defaults(debug=LyX.default_debug__, cjk_encoding = '') - parser.add_option("-d", "--debug", type="int", + parser.add_argument("-d", "--debug", type=int, dest="debug", help="level=0..2 (O_ quiet, 10_verbose) default: 2") - parser.add_option("-q", "--quiet", + parser.add_argument("-q", "--quiet", action="store_const", const=0, dest="debug") - parser.add_option("-v", "--verbose", + parser.add_argument("-v", "--verbose", action="store_const", const=1, dest="debug") - parser.add_option("--noisy", + parser.add_argument("--noisy", action="store_const", const=10, dest="debug") - parser.add_option("-c", "--encoding", dest="cjk_encoding", + parser.add_argument("-c", "--encoding", type=cmd_arg, dest="cjk_encoding", help="files in format 413 and lower are read and" " written in the format of CJK-LyX." "If encoding is not given or 'auto' the encoding" "is determined from the locale.") - parser.add_option("-e", "--err", dest="error", + parser.add_argument("-e", "--err", type=cmd_arg, dest="error", help= "file name of the error file else goes to stderr") - parser.add_option("-o", "--output", + parser.add_argument("-o", "--output", type=cmd_arg, dest="output", help= "name of the output file else goes to stdout") - parser.add_option("-t", "--to", dest= "end_format", + parser.add_argument("-t", "--to", type=cmd_arg, dest= "end_format", help= "destination file format, default (latest)") - parser.add_option("-V", "--final_version", dest= "final_version", + parser.add_argument("-V", "--final_version", type=cmd_arg, dest= "final_version", help= "destination version, default (latest)") - parser.add_option("-l", "--list", action="store_true", + parser.add_argument("-l", "--list", action="store_true", help = "list all available formats and supported versions") - parser.add_option("-n", "--try-hard", action="store_true", + parser.add_argument("-n", "--try-hard", action="store_true", help = "try hard (ignore any convertion errors)") - parser.add_option("-s", "--systemlyxdir", dest= "systemlyxdir", + parser.add_argument("-s", "--systemlyxdir", type=cmd_arg, dest= "systemlyxdir", help= "LyX system directory for conversion from version 489 or older") + parser.add_argument('--version', action='version', version="""lyx2lyx, version %s +Copyright (C) 2011 The LyX Team, José Matos and Dekel Tsur""" % LyX.version__) + parser.add_argument("input", nargs='?', type=cmd_arg, default=None) - (options, args) = parser.parse_args() - if args: - options.input = args[0] - else: - options.input = None + options = parser.parse_args() if options.list: sys.stderr.write(LyX.format_info()) diff --git a/lib/lyx2lyx/lyx_1_4.py b/lib/lyx2lyx/lyx_1_4.py index c1a4591b54..29abc53bfa 100644 --- a/lib/lyx2lyx/lyx_1_4.py +++ b/lib/lyx2lyx/lyx_1_4.py @@ -1893,8 +1893,7 @@ def convert_graphics(document): return i = i + 1 filename = document.body[j].split()[1] - absname = os.path.normpath(os.path.join(document.dir, filename)) - if document.input == stdin and not os.path.isabs(filename): + if document.dir == u'' and not os.path.isabs(filename): # We don't know the directory and cannot check the document. # We could use a heuristic and take the current directory, # and we could try to find out if documentname has an extension, @@ -1905,6 +1904,7 @@ def convert_graphics(document): You may need to correct the document manually or run lyx2lyx again with the .lyx document as commandline argument.""" % filename) continue + absname = os.path.normpath(os.path.join(document.dir, filename)) # This needs to be the same algorithm as in pre 233 insetgraphics if access(absname, F_OK): continue diff --git a/lib/lyx2lyx/lyx_2_2.py b/lib/lyx2lyx/lyx_2_2.py index 519c8b5d9c..34490d8dfc 100644 --- a/lib/lyx2lyx/lyx_2_2.py +++ b/lib/lyx2lyx/lyx_2_2.py @@ -39,10 +39,6 @@ from parser_tools import find_token, find_token_backwards, find_re, \ find_end_of_inset, find_end_of_layout, find_nonempty_line, \ get_containing_layout, get_value, check_token -# Provide support for both python 2 and 3 -PY2 = sys.version_info[0] == 2 -# End of code to support for both python 2 and 3 - #################################################################### # Private helper functions @@ -1135,11 +1131,11 @@ def convert_origin(document): if i == -1: document.warning("Malformed LyX document: No \\textclass!!") return - if document.dir == "": - origin = "stdin" + if document.dir == u'': + origin = u'stdin' else: - relpath = '' - if document.systemlyxdir and document.systemlyxdir != '': + relpath = u'' + if document.systemlyxdir and document.systemlyxdir != u'': try: if os.path.isabs(document.dir): absdir = os.path.normpath(document.dir) @@ -1150,16 +1146,14 @@ def convert_origin(document): else: abssys = os.path.normpath(os.path.abspath(document.systemlyxdir)) relpath = os.path.relpath(absdir, abssys) - if relpath.find('..') == 0: - relpath = '' + if relpath.find(u'..') == 0: + relpath = u'' except: - relpath = '' - if relpath == '': - origin = document.dir.replace('\\', '/') + '/' + relpath = u'' + if relpath == u'': + origin = document.dir.replace(u'\\', u'/') + u'/' else: - origin = os.path.join("/systemlyxdir", relpath).replace('\\', '/') + '/' - if os.name != 'nt' and PY2: - origin = unicode(origin, sys.getfilesystemencoding()) + origin = os.path.join(u"/systemlyxdir", relpath).replace(u'\\', u'/') + u'/' document.header[i:i] = ["\\origin " + origin]