Merge branch 'master' of git.lyx.org:lyx

development/FORMAT

@@ -11,6 +11,14 @@ adjustments are made to tex2lyx and bugs are fixed in lyx2lyx.
 
 -----------------------
 
+2013-02-18 Julien Rioux <jrioux@lyx.org>
+	* Format incremented to 463:
+	  - Use the LyX name of encodings instead of the LaTeX names.
+	    The LyX name must be unique, while the name used by LaTeX
+	    not necessarily, e.g. different packages might implement
+	    support for the same encoding.
+	  - Add utf8-platex encoding.
+
 2013-02-15 Jürgen Spitzmüller <spitz@lyx.org>
 	* Format incremented to 462: Support for recent libertine LaTeX fonts
 	  a.) Support Libertine Mono:

lib/encodings

@@ -3,7 +3,7 @@
 
 # Note that you can only add singlebyte encodings to this file.
 # LyX does not support the output of multibyte encodings (e.g. utf16).
-# It does support singlebyte encodings with variable with (e.g. utf8).
+# It does support singlebyte encodings with variable width (e.g. utf8).
 # These are marked with the "variable" keyword.
 # Fixed width encodings are marked with the "fixed" keyword.
 # The code points of TeX control characters like {, } and \ can occur in the
@@ -11,7 +11,20 @@
 # set as document encodings and are marked with the "variableunsafe" keyword.
 # They are only needed for proper tex2lyx import.
 
-# Syntax: Encoding <LyX name> <LaTeX name> <GUI name> <iconv name> fixed|variable|variableunsafe <package> End
+# Most encodings require loading a latex package such as "inputenc" or "CJK".
+# There is no "japanese" latex package, rather this keyword indicates to LyX
+# to switch the buffer format and use platex instead of standard (pdf)latex.
+# In this case, TeX control characters in high bytes is not a problem.
+# The invocation is platex -kanji=<LaTeX name>
+
+# Syntax: Encoding <LyX name> <LaTeX name> <GUI name> <iconv name> <width> <package> End
+
+# LyX name:    Name used by the file format and in lib/languages. Must be unique!
+# LaTeX name:  Used in the latex export or passed to platex as command-line switch.
+# GUI name:    Displayed in document settings.
+# iconv name:  Used by iconv.
+# width:       One of fixed, variable, or variableunsafe (see above).
+# package:     One of none, inputenc, CJK, or japanese (see above).
 
 # encodings used by inputenc.sty
 
@@ -116,7 +129,7 @@ End
 Encoding cp1257 cp1257 "Baltic (CP 1257)" CP1257 fixed inputenc
 End
 
-Encoding koi8 koi8-r "Cyrillic (KOI8-R)" KOI8-R fixed inputenc
+Encoding koi8-r koi8-r "Cyrillic (KOI8-R)" KOI8-R fixed inputenc
 End
 
 Encoding koi8-u koi8-u "Cyrillic (KOI8-U)" KOI8-U fixed inputenc
@@ -184,12 +197,13 @@ End
 
 # Traditional Japanese TeX programs require the japanese package.
 # that is incompatible with CJK and inputenc.
-Encoding euc-jp-plain euc "Japanese (non-CJK) (EUC-JP)" EUC-JP variable japanese
+Encoding euc-jp-platex euc "Japanese (pLaTeX) (EUC-JP)" EUC-JP variable japanese
 End
-Encoding jis-plain jis "Japanese (non-CJK) (JIS)" ISO-2022-JP variable japanese
+Encoding jis-platex jis "Japanese (pLaTeX) (JIS)" ISO-2022-JP variable japanese
 End
-# FIXME: Should use variableunsafe (would be a file format change)
+Encoding shift-jis-platex sjis "Japanese (pLaTeX) (SJIS)" CP932 variable japanese
-Encoding shift-jis-plain sjis "Japanese (non-CJK) (SJIS)" CP932 variable japanese
+End
+Encoding utf8-platex utf8 "Japanese (pLaTeX) (UTF8)" UTF-8 variable japanese
 End
 
 # This one needs hardcoded support, since the inputenc package does not know
@@ -207,4 +221,3 @@ End
 # Pure 7bit ASCII encoding (partially hardcoded in LyX)
 Encoding ascii ascii "ASCII" ascii fixed none
 End
-

lib/languages

@@ -576,7 +576,7 @@ End
 Language japanese
 	GuiName          "Japanese"
 	BabelName        japanese
-	Encoding         jis-plain
+	Encoding         jis-platex
 	LangCode         ja_JP
 	AsBabelOptions   true
 	Requires         japanese
@@ -773,7 +773,7 @@ Language russian
 	BabelName        russian
 	PolyglossiaName  russian
 	QuoteStyle       french
-	Encoding         koi8
+	Encoding         koi8-r
 	LangCode         ru_RU
 End
 

lib/lyx2lyx/lyx_2_1.py

@@ -3546,6 +3546,90 @@ def revert_newframes(document):
         document.body[i : i + 1] = subst
         i = j
 
+# known encodings that do not change their names (same LyX and LaTeX names)
+known_enc_tuple = ("auto", "default", "ansinew", "applemac", "armscii8", "ascii",
+    "cp437", "cp437de", "cp850", "cp852", "cp855", "cp858", "cp862", "cp865", "cp866",
+    "cp1250", "cp1251", "cp1252", "cp1255", "cp1256", "cp1257", "koi8-r", "koi8-u",
+    "pt154", "pt254", "tis620-0", "utf8", "utf8x", "utf8-plain")
+
+def convert_encodings(document):
+    "Use the LyX names of the encodings instead of the LaTeX names."
+    LaTeX2LyX_enc_dict = {
+        "8859-6":     "iso8859-6",
+        "8859-8":     "iso8859-8",
+        "Bg5":        "big5",
+        "euc":        "euc-jp-platex",
+        "EUC-JP":     "euc-jp",
+        "EUC-TW":     "euc-tw",
+        "GB":         "euc-cn",
+        "GBK":        "gbk",
+        "iso88595":   "iso8859-5",
+        "iso-8859-7": "iso8859-7",
+        "JIS":        "jis",
+        "jis":        "jis-platex",
+        "KS":         "euc-kr",
+        "l7xenc":     "iso8859-13",
+        "latin1":     "iso8859-1",
+        "latin2":     "iso8859-2",
+        "latin3":     "iso8859-3",
+        "latin4":     "iso8859-4",
+        "latin5":     "iso8859-9",
+        "latin9":     "iso8859-15",
+        "latin10":    "iso8859-16",
+        "SJIS":       "shift-jis",
+        "sjis":       "shift-jis-platex",
+        "UTF8":       "utf8-cjk"
+    }
+    i = find_token(document.header, "\\inputencoding" , 0)
+    if i == -1:
+        return
+    val = get_value(document.header, "\\inputencoding", i)
+    if val in LaTeX2LyX_enc_dict.keys():
+        document.header[i] = "\\inputencoding %s" % LaTeX2LyX_enc_dict[val]
+    elif val not in known_enc_tuple:
+        document.warning("Ignoring unknown input encoding: `%s'" % val)
+
+
+def revert_encodings(document):
+    """Revert to using the LaTeX names of the encodings instead of the LyX names.
+    Also revert utf8-platex to sjis, the language default when using Japanese.
+    """
+    LyX2LaTeX_enc_dict = {
+        "big5":             "Bg5",
+        "euc-cn":           "GB",
+        "euc-kr":           "KS",
+        "euc-jp":           "EUC-JP",
+        "euc-jp-platex":    "euc",
+        "euc-tw":           "EUC-TW",
+        "gbk":              "GBK",
+        "iso8859-1":        "latin1",
+        "iso8859-2":        "latin2",
+        "iso8859-3":        "latin3",
+        "iso8859-4":        "latin4",
+        "iso8859-5":        "iso88595",
+        "iso8859-6":        "8859-6",
+        "iso8859-7":        "iso-8859-7",
+        "iso8859-8":        "8859-8",
+        "iso8859-9":        "latin5",
+        "iso8859-13":       "l7xenc",
+        "iso8859-15":       "latin9",
+        "iso8859-16":       "latin10",
+        "jis":              "JIS",
+        "jis-platex":       "jis",
+        "shift-jis":        "SJIS",
+        "shift-jis-platex": "sjis",
+        "utf8-cjk":         "UTF8",
+        "utf8-platex":      "sjis"
+    }
+    i = find_token(document.header, "\\inputencoding" , 0)
+    if i == -1:
+        return
+    val = get_value(document.header, "\\inputencoding", i)
+    if val in LyX2LaTeX_enc_dict.keys():
+        document.header[i] = "\\inputencoding %s" % LyX2LaTeX_enc_dict[val]
+    elif val not in known_enc_tuple:
+        document.warning("Ignoring unknown input encoding: `%s'" % val)
+
 
 def revert_IEEEtran_3(document):
   '''
@@ -3699,10 +3783,12 @@ convert = [
            [459, []],
            [460, []],
            [461, []],
-           [462, []]
+           [462, []],
+           [463, [convert_encodings]],
           ]
 
 revert =  [
+           [462, [revert_encodings]],
            [461, [revert_new_libertines]],
            [460, [revert_kurier_fonts]],
            [459, [revert_IEEEtran_3]],

src/BufferParams.cpp

@@ -2786,7 +2786,7 @@ void BufferParams::writeEncodingPreamble(otexstream & os,
 			// do not load inputenc if japanese is used
 			if (features.isRequired("japanese"))
 				break;
-			os << "\\usepackage[" << from_ascii(inputenc)
+			os << "\\usepackage[" << from_ascii(encoding().latexName())
 			   << "]{inputenc}\n";
 			break;
 		case Encoding::CJK:
@@ -2913,10 +2913,10 @@ Encoding const & BufferParams::encoding() const
 	// This check will not work with XeTeX/LuaTeX and tex fonts.
 	// Thus we have to reset the encoding in Buffer::makeLaTeXFile.
 	if (useNonTeXFonts)
-		return *(encodings.fromLaTeXName("utf8-plain"));
+		return *(encodings.fromLyXName("utf8-plain"));
 	if (inputenc == "auto" || inputenc == "default")
 		return *language->encoding();
-	Encoding const * const enc = encodings.fromLaTeXName(inputenc);
+	Encoding const * const enc = encodings.fromLyXName(inputenc);
 	if (enc)
 		return *enc;
 	LYXERR0("Unknown inputenc value `" << inputenc

src/BufferParams.h

@@ -284,10 +284,10 @@ public:
 	IndicesList & indiceslist();
 	IndicesList const & indiceslist() const;
 	/**
-	 * The input encoding for LaTeX. This can be one of
+	 * The LyX name of the input encoding for LaTeX. This can be one of
 	 * - \c auto: find out the input encoding from the used languages
 	 * - \c default: ditto
-	 * - any encoding supported by the inputenc package
+	 * - any encoding defined in the file lib/encodings
 	 * The encoding of the LyX file is always utf8 and has nothing to
 	 * do with this setting.
 	 * The difference between \c auto and \c default is that \c auto also

src/frontends/qt4/GuiDocument.cpp

@@ -2586,7 +2586,7 @@ void GuiDocument::applyView()
 			for (; it != end; ++it) {
 				if (qt_(it->guiName()) == enc_gui &&
 				    !it->unsafe()) {
-					bp_.inputenc = it->latexName();
+					bp_.inputenc = it->name();
 					found = true;
 					break;
 				}
@@ -2993,7 +2993,7 @@ void GuiDocument::paramsToDialog()
 			Encodings::const_iterator it = encodings.begin();
 			Encodings::const_iterator const end = encodings.end();
 			for (; it != end; ++it) {
-				if (it->latexName() == bp_.inputenc &&
+				if (it->name() == bp_.inputenc &&
 				    !it->unsafe()) {
 					enc_gui = it->guiName();
 					break;

src/support/unicode.cpp

@@ -377,7 +377,6 @@ int max_encoded_bytes(std::string const & encoding)
 	// The CJK encodings use (different) multibyte representation as well.
 	// All other encodings encode one UCS4 code point in one byte
 	// (and can therefore only encode a subset of UCS4)
-	// Note that BIG5 and SJIS do not work with LaTeX (see lib/encodings).
 	// Furthermore, all encodings that use shifting (like SJIS) do not work with
 	// iconv_codecvt_facet.
 	if (encoding == "UTF-8" ||

src/tex2lyx/Preamble.cpp

@@ -805,7 +805,7 @@ void Preamble::handle_package(Parser &p, string const & name,
 				Encoding const * const enc = encodings.fromIconvName(
 					p.getEncoding(), Encoding::japanese, false);
 				if (enc)
-					h_inputencoding = enc->latexName();
+					h_inputencoding = enc->name();
 				is_nonCJKJapanese = true;
 				// in this case babel can be removed from the preamble
 				registerAutomaticallyLoadedPackage("babel");
@@ -844,7 +844,7 @@ void Preamble::handle_package(Parser &p, string const & name,
 	}
 
 	else if (name == "CJKutf8") {
-		h_inputencoding = "UTF8";
+		h_inputencoding = "utf8-cjk";
 		p.setEncoding("UTF-8");
 		registerAutomaticallyLoadedPackage("CJKutf8");
 	}
@@ -863,14 +863,22 @@ void Preamble::handle_package(Parser &p, string const & name,
 		// h_inputencoding is only set when there is not more than one
 		// inputenc option because otherwise h_inputencoding must be
 		// set to "auto" (the default encoding of the document language)
-		// Therefore check for the "," character.
+		// Therefore check that exactly one option is passed to inputenc.
 		// It is also only set when there is not more than one babel
 		// language option.
-		if (opts.find(",") == string::npos && one_language == true)
+		if (!options.empty()) {
-			h_inputencoding = opts;
+			string const encoding = options.back();
-		if (!options.empty())
+			Encoding const * const enc = encodings.fromLaTeXName(
-			p.setEncoding(options.back(), Encoding::inputenc);
+				encoding, Encoding::inputenc, true);
-		options.clear();
+			if (!enc)
+				cerr << "Unknown encoding " << encoding << ". Ignoring." << std::endl;
+			else {
+				if (!enc->unsafe() && options.size() == 1 && one_language == true)
+					h_inputencoding = enc->name();
+				p.setEncoding(enc->iconvName());
+			}
+			options.clear();
+		}
 	}
 
 	else if (name == "srcltx") {
@@ -1624,8 +1632,15 @@ void Preamble::parse(Parser & p, string const & forceclass,
 
 		else if (t.cs() == "inputencoding") {
 			string const encoding = p.getArg('{','}');
-			h_inputencoding = encoding;
+			Encoding const * const enc = encodings.fromLaTeXName(
-			p.setEncoding(encoding, Encoding::inputenc);
+				encoding, Encoding::inputenc, true);
+			if (!enc)
+				cerr << "Unknown encoding " << encoding << ". Ignoring." << std::endl;
+			else {
+				if (!enc->unsafe())
+					h_inputencoding = enc->name();
+				p.setEncoding(enc->iconvName());
+			}
 		}
 
 		else if (t.cs() == "newenvironment") {

src/tex2lyx/TODO.txt

@@ -87,10 +87,10 @@ Format LaTeX feature                        LyX feature
        \usepackage[scale|scaled=$val]{biolinum-type1}
                                             \font_sans
                                             \font_sf_scale
+463
 
 
 General
 
 * Use the language information provided by Language.cpp and the languages file (for babel/lyx/polyglossia name, quote style etc.)
   instead of hardcoding this information in Preamble.cpp.
-

src/tex2lyx/tex2lyx.cpp

@@ -840,8 +840,8 @@ bool tex2lyx(idocstream & is, ostream & os, string encoding)
 		if (preamble.inputencoding() == "auto")
 			encoding = "ISO8859-1";
 		else {
-			Encoding const * const enc = encodings.fromLaTeXName(
+			Encoding const * const enc = encodings.fromLyXName(
-				preamble.inputencoding(), Encoding::any, true);
+				preamble.inputencoding(), true);
 			encoding = enc->iconvName();
 		}
 	}

src/version.h

@@ -30,8 +30,8 @@ extern char const * const lyx_version_info;
 
 // Do not remove the comment below, so we get merge conflict in
 // independent branches. Instead add your own.
-#define LYX_FORMAT_LYX 462 // spitz: support for the newest libertine fonts
+#define LYX_FORMAT_LYX 463 // jrioux: encodings renaming
-#define LYX_FORMAT_TEX2LYX 462 // spitz: support for the newest libertine fonts
+#define LYX_FORMAT_TEX2LYX 463 // jrioux: encodings renaming
 
 #if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX
 #ifndef _MSC_VER