Add some more LaTeX encodings, fix thailatex and display all available

encodings in the preferences dialog

	* src/LaTeXFeatures.C: special treatment for tis620-0 encoding
	* src/bufferparams.C: ditto
	* src/output_latex.C: ditto

	* src/encoding.[Ch]: Add the possibility to iterate over all encodings

	* src/paragraph_pimpl.C: Add latin10 and cp858 to Euro treatment

	* src/buffer.C: Update format number

	* src/frontends/qt4/QDocumentDialog.C: Don't hardcode available
	encodings

	* lib/lyx2lyx/LyX.py,
	  lib/lyx2lyx/lyx_1_5.py: implement conversion from 256 to 255

	* lib/encodings: Add some encodings, fix all "unknown" entries

	* development/FORMAT: Document file format change


git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@16418 a592a061-630c-0410-9148-cb99ea01b6c8

development/FORMAT

@@ -1,6 +1,11 @@
 LyX file-format changes
 -----------------------§
 
+2006-12-22  Georg Baum  <Georg.Baum@post.rwth-aachen.de>
+
+	* format incremented to 256: allow some new inputenc settings.
+	For the complete list, see lib/lyx2lyx/lyx_1_5.py.
+
 2006-11-25  Georg Baum  <Georg.Baum@post.rwth-aachen.de>
 
 	* format incremented to 255: new insets for \clearpage and

lib/encodings

@@ -1,8 +1,5 @@
 # FIXME: Have a look at the encodings known by the inputenc package and add
 # missing ones. Caution: File format change!
-# FIXME: Find out whether this file is used for more than LaTeX file.
-# generation. If not it doews not make make sense to have encodings with
-# "unknown" LaTeX name.
 
 # Order of names: LyX name LaTeX name iconv name
 
@@ -24,35 +21,91 @@ End
 Encoding iso8859-5 iso88595 ISO-8859-5
 End
 
-Encoding iso8859-6 unknown ISO-8859-6
+# Not standard, see http://tug.ctan.org/tex-archive/language/arabic/arabi/arabi/texmf/latex/arabi/
+Encoding iso8859-6 8859-6 ISO-8859-6
 End
 
 Encoding iso8859-7 iso-8859-7 ISO-8859-7
 End
 
+Encoding iso8859-8 8859-8 ISO-8859-8
+End
+
 Encoding iso8859-9 latin5 ISO-8859-9
 End
 
+# Not standard, see http://www.vtex.lt/tex/littex/index.html
 Encoding iso8859-13 l7xenc ISO-8859-13
 End
 
 Encoding iso8859-15 latin9 ISO-8859-15
 End
 
-Encoding cp1255 cp1255 CP1255
+Encoding iso8859-16 latin10 ISO-8859-16
+End
+
+Encoding cp437 cp437 CP437
+End
+
+# cp437, but on position 225 is sz instead of beta
+Encoding cp437de cp437de CP437
+End
+
+Encoding cp850 cp850 CP850
+End
+
+Encoding cp852 cp852 CP852
+End
+
+Encoding cp855 cp855 CP855
+End
+
+Encoding cp858 cp858 CP858
+End
+
+Encoding cp862 cp862 CP862
+End
+
+Encoding cp865 cp865 CP865
+End
+
+Encoding cp866 cp866 CP866
+End
+
+Encoding cp1250 cp1250 CP1250
 End
 
 Encoding cp1251 cp1251 CP1251
 End
 
+Encoding cp1252 cp1252 CP1252
+End
+
+Encoding cp1255 cp1255 CP1255
+End
+
+# Not standard, see http://tug.ctan.org/tex-archive/language/arabic/arabi/arabi/texmf/latex/arabi/
+Encoding cp1256 cp1256 CP1256
+End
+
+Encoding cp1257 cp1257 CP1257
+End
+
 Encoding koi8 koi8-r KOI8-R
 End
 
 Encoding koi8-u koi8-u KOI8-U
 End
 
-Encoding tis620-0 unknown TIS-620-0
+# This one needs hardcoded support, since the inputenc package does not know
+# tis620-0, and thailatex sets up babel directly to use tis620-0, so the value
+# for inputenc is never output to .tex files (but needed for the hardcoded
+# tis620-0 support).
+Encoding tis620-0 tis620-0 TIS620-0
 End
 
 Encoding pt154 pt154 PT154
 End
+
+Encoding pt254 pt254 PT254
+End

lib/lyx2lyx/LyX.py

@@ -73,7 +73,7 @@ format_relation = [("0_06",    [200], generate_minor_versions("0.6" , 4)),
                    ("1_2",     [220], generate_minor_versions("1.2" , 4)),
                    ("1_3",     [221], generate_minor_versions("1.3" , 7)),
                    ("1_4", range(222,246), generate_minor_versions("1.4" , 3)),
-                   ("1_5", range(246,256), generate_minor_versions("1.5" , 0))]
+                   ("1_5", range(246,257), generate_minor_versions("1.5" , 0))]
 
 
 def formats_list():

lib/lyx2lyx/lyx_1_5.py

@@ -646,6 +646,21 @@ def revert_cleardoublepage(document):
     i = i + 1
 
 
+def revert_encodings(document):
+    " Set new encodings to auto. "
+    encodings = ["8859-6", "8859-8", "cp437", "cp437de", "cp850", "cp852",
+                 "cp855", "cp858", "cp862", "cp865", "cp866", "cp1250",
+                 "cp1252", "cp1256", "cp1257", "latin10", "pt254", "tis620-0"]
+    i = find_token(document.header, "\\inputencoding", 0)
+    if i == -1:
+        document.header.append("\\inputencoding auto")
+    else:
+        inputenc = get_value(document.header, "\\inputencoding", i)
+        if inputenc in encodings:
+            document.header[i] = "\\inputencoding auto"
+    document.inputencoding = get_value(document.header, "\\inputencoding", 0)
+
+
 ##
 # Conversion hub
 #
@@ -660,9 +675,11 @@ convert = [[246, []],
            [252, [convert_commandparams, convert_bibitem]],
            [253, []],
            [254, [convert_esint]],
-           [255, []]]
+           [255, []],
+           [256, []]]
 
-revert =  [[254, [revert_clearpage, revert_cleardoublepage]],
+revert =  [[255, [revert_encodings]],
+           [254, [revert_clearpage, revert_cleardoublepage]],
            [253, [revert_esint]],
            [252, [revert_nomenclature, revert_printnomenclature]],
            [251, [revert_commandparams]],

src/LaTeXFeatures.C

@@ -221,7 +221,11 @@ set<string> LaTeXFeatures::getEncodingSet(string const & doc_encoding) const
 	LanguageList::const_iterator it  = UsedLanguages_.begin();
 	LanguageList::const_iterator end = UsedLanguages_.end();
 	for (; it != end; ++it)
-		if ((*it)->encoding()->latexName() != doc_encoding)
+		// thailatex does not use the inputenc package, but sets up
+		// babel directly for tis620-0 encoding, therefore we must
+		// not add tis620-0 to the encoding set.
+		if ((*it)->encoding()->latexName() != doc_encoding &&
+		    (*it)->encoding()->name() != "tis620-0")
 			encodings.insert((*it)->encoding()->latexName());
 	return encodings;
 }

src/buffer.C

@@ -141,7 +141,7 @@ using std::string;
 
 namespace {
 
-int const LYX_FORMAT = 255;
+int const LYX_FORMAT = 256;
 
 } // namespace anon
 

src/bufferparams.C

@@ -851,14 +851,28 @@ bool BufferParams::writeLaTeX(odocstream & os, LaTeXFeatures & features,
 		std::set<string> encodings =
 			features.getEncodingSet(doc_encoding);
 
-		os << "\\usepackage[";
-		std::set<string>::const_iterator it = encodings.begin();
-		std::set<string>::const_iterator const end = encodings.end();
-		for (; it != end; ++it)
-			os << from_ascii(*it) << ',';
-		os << from_ascii(doc_encoding) << "]{inputenc}\n";
-		texrow.newline();
-	} else if (inputenc != "default") {
+		// thailatex does not use the inputenc package, but sets up
+		// babel directly for tis620-0 encoding, therefore we must
+		// not request inputenc for tis620-0 encoding
+		if (!encodings.empty() || doc_encoding != "tis620-0") {
+			os << "\\usepackage[";
+			std::set<string>::const_iterator it = encodings.begin();
+			std::set<string>::const_iterator const end = encodings.end();
+			if (it != end) {
+				os << from_ascii(*it);
+				++it;
+			}
+			for (; it != end; ++it)
+				os << ',' << from_ascii(*it);
+			if (doc_encoding != "tis620-0") {
+				if (!encodings.empty())
+					os << ',';
+				os << from_ascii(doc_encoding);
+			}
+			os << "]{inputenc}\n";
+			texrow.newline();
+		}
+	} else if (inputenc != "default" && inputenc != "tis620-0") {
 		os << "\\usepackage[" << from_ascii(inputenc)
 		   << "]{inputenc}\n";
 		texrow.newline();

src/encoding.h

@@ -52,6 +52,16 @@ class Encodings {
 public:
 	///
 	typedef std::map<std::string, Encoding> EncodingList;
+	/// iterator to iterate over all encodings.
+	/// We hide the fact that our encoding list is implemented as a map.
+	class const_iterator : public EncodingList::const_iterator {
+		typedef EncodingList::const_iterator base;
+	public:
+		const_iterator() : base() {}
+		const_iterator(base const & b) : base(b) {}
+		Encoding const & operator*() const { return base::operator*().second; }
+		Encoding const * operator->() const { return &(base::operator*().second); }
+	};
 	///
 	Encodings();
 	///
@@ -61,6 +71,11 @@ public:
 	/// Get encoding from LaTeX name \p name
 	Encoding const * getFromLaTeXName(std::string const & name) const;
 
+	///
+	const_iterator begin() const { return encodinglist.begin(); }
+	///
+	const_iterator end() const { return encodinglist.end(); }
+
 	///
 	enum Letter_Form {
 		///

src/frontends/qt4/QDocumentDialog.C

@@ -25,6 +25,7 @@
 #include "qt_helpers.h"
 
 #include "bufferparams.h"
+#include "encoding.h"
 #include "gettext.h"
 #include "helper_funcs.h" // getSecond()
 #include "language.h"
@@ -56,18 +57,6 @@ namespace lyx {
 namespace frontend {
 
 
-namespace {
-
-// FIXME: This list is incomplete. It should not be hardcoded but come from
-// the available encodings in src/encodings.C
-char const * encodings[] = { "LaTeX default", "latin1", "latin2",
-	"latin3", "latin4", "latin5", "latin9",
-	"koi8-r", "koi8-u", "cp866", "cp1251",
-	"iso88595", "pt154", "utf8", 0
-};
-
-}
-
 QDocumentDialog::QDocumentDialog(QDocument * form)
 	: form_(form),
 	lang_(getSecond(getLanguageData(false)))
@@ -298,10 +287,13 @@ QDocumentDialog::QDocumentDialog(QDocument * form)
 			toqstr(lit->first));
 	}
 
-	int k = 0;
-	while (encodings[k]) {
-		langModule->encodingCO->addItem(qt_(encodings[k++]));
-	}
+	// Always put the default encoding in the first position.
+	// It is special because the displayed text is translated.
+	langModule->encodingCO->addItem(qt_("LaTeX default"));
+	Encodings::const_iterator it = encodings.begin();
+	Encodings::const_iterator const end = encodings.end();
+	for (; it != end; ++it)
+		langModule->encodingCO->addItem(toqstr(it->latexName()));
 
 	langModule->quoteStyleCO->addItem(qt_("``text''"));
 	langModule->quoteStyleCO->addItem(qt_("''text''"));
@@ -669,11 +661,11 @@ void QDocumentDialog::apply(BufferParams & params)
 		params.inputenc = "auto";
 	} else {
 		int i = langModule->encodingCO->currentIndex();
-		if (i == 0) {
+		if (i == 0)
 			params.inputenc = "default";
-		} else {
-			params.inputenc = encodings[i];
-		}
+		else
+			params.inputenc =
+				fromqstr(langModule->encodingCO->currentText());
 	}
 
 	InsetQuotes::quote_language lga = InsetQuotes::EnglishQ;
@@ -956,16 +948,13 @@ void QDocumentDialog::update(BufferParams const & params)
 		if (params.inputenc == "default") {
 			langModule->encodingCO->setCurrentIndex(0);
 		} else {
-			int i = 0;
-			while (encodings[i]) {
-				if (encodings[i] == params.inputenc) {
-					langModule->encodingCO->setCurrentIndex(i);
-					break;
-				}
-				++i;
-			}
-			// FIXME: possible data loss because of encodings is
-			// incomplete
+			int const i = langModule->encodingCO->findText(
+					toqstr(params.inputenc));
+			if (i >= 0)
+				langModule->encodingCO->setCurrentIndex(i);
+			else
+				// unknown encoding. Set to default.
+				langModule->defaultencodingCB->setChecked(true);
 		}
 	}
 

src/lyx_cb.C

@@ -59,7 +59,6 @@
 
 namespace lyx {
 
-using support::addName;
 using support::bformat;
 using support::FileFilterList;
 using support::FileName;

src/output_latex.C

@@ -294,8 +294,15 @@ TeXOnePar(Buffer const & buf,
 		}
 	}
 
+	// FIXME thailatex does not support the inputenc package, so we
+	// ignore switches from/to tis620-0 encoding here. This does of
+	// course only work as long as the non-thai text contains ASCII
+	// only, but it is the best we can do.
+	bool const use_thailatex = (language->encoding()->name() == "tis620-0" ||
+	                            previous_language->encoding()->name() == "tis620-0");
 	if (bparams.inputenc == "auto" &&
-	    language->encoding() != previous_language->encoding()) {
+	    language->encoding() != previous_language->encoding() &&
+	    !use_thailatex) {
 		ucs4 << "\\inputencoding{"
 		     << from_ascii(language->encoding()->latexName())
 		     << "}\n";
@@ -307,7 +314,8 @@ TeXOnePar(Buffer const & buf,
 	odocstringstream par_stream;
 	bool const change_encoding = !runparams_in.dryrun &&
 			bparams.inputenc == "auto" &&
-			language->encoding() != doc_language->encoding();
+			language->encoding() != doc_language->encoding() &&
+			!use_thailatex;
 	// don't trigger the copy ctor because it's private on msvc 
 	odocstream & os = *(change_encoding ? &par_stream : &ucs4);
 

src/paragraph_pimpl.C

@@ -663,7 +663,9 @@ void Paragraph::Pimpl::simpleTeXSpecialChars(Buffer const & buf,
 		case 0x20ac:    // EURO SIGN
 			if (isEncoding(bparams, font, "latin9")
 			    || isEncoding(bparams, font, "cp1251")
-			    || isEncoding(bparams, font, "utf8")) {
+			    || isEncoding(bparams, font, "utf8")
+			    || isEncoding(bparams, font, "latin10")
+			    || isEncoding(bparams, font, "cp858")) {
 				os.put(c);
 			} else {
 				os << "\\texteuro{}";