First version of separate translation machinery for strings that go into

exported documents.
To update the translations from the .po files, run
make ../lib/layouttranslations
from po.


git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@38016 a592a061-630c-0410-9148-cb99ea01b6c8

development/scons/scons_manifest.py

@@ -1288,6 +1288,7 @@ lib_files = Split('''
     encodings
     external_templates
     languages
+    layouttranslations
     symbols
     syntax.default
     unicodesymbols

lib/Makefile.am

@@ -5,7 +5,7 @@ SUBDIRS = doc lyx2lyx
 CHMOD = chmod
 
 dist_pkgdata_DATA = CREDITS chkconfig.ltx external_templates encodings \
-  languages symbols syntax.default unicodesymbols
+	layouttranslations languages symbols syntax.default unicodesymbols
 
 # Note that we "chmod 755" manually this file in install-data-hook.
 dist_pkgdata_PYTHON = configure.py 

po/Rules-lyx

@@ -30,6 +30,11 @@ layouts_l10n.pot: $(top_srcdir)/lib/layouts/*.layout $(top_srcdir)/lib/layouts/*
 	LC_ALL=C ; export LC_ALL ; \
 	python $(srcdir)/lyx_pot.py -b $(top_srcdir) -o $@ -t layouts ${top_srcdir}/lib/layouts/*.layout ${top_srcdir}/lib/layouts/*.inc ${top_srcdir}/lib/layouts/*.module
 
+$(top_srcdir)/lib/layouttranslations: $(POFILES) $(top_srcdir)/lib/layouts/*.layout \
+		$(top_srcdir)/lib/layouts/*.inc $(top_srcdir)/lib/layouts/*.module
+	LC_ALL=C ; export LC_ALL ; \
+	python $(srcdir)/lyx_pot.py -b $(top_srcdir) -o $@ -t layouttranslations ${top_srcdir}/lib/layouts/*.layout ${top_srcdir}/lib/layouts/*.inc ${top_srcdir}/lib/layouts/*.module
+
 languages_l10n.pot: $(top_srcdir)/lib/languages
 	python $(srcdir)/lyx_pot.py -b $(top_srcdir) -o $@ -t languages ${top_srcdir}/lib/languages
 

po/lyx_pot.py

@@ -17,6 +17,8 @@
 # to output in gettext .pot format.
 #
 import sys, os, re, getopt
+if sys.version_info < (2, 4, 0):
+    from sets import Set as set
 
 def relativePath(path, base):
     '''return relative path from top source dir'''
@@ -74,7 +76,7 @@ def ui_l10n(input_files, output, base):
     output.close()
 
 
-def layouts_l10n(input_files, output, base):
+def layouts_l10n(input_files, output, base, layouttranslations):
     '''Generate pot file from lib/layouts/*.{layout,inc,module}'''
     out = open(output, 'w')
     Style = re.compile(r'^Style\s+(.*)', re.IGNORECASE)
@@ -95,11 +97,26 @@ def layouts_l10n(input_files, output, base):
     CounterFormat = re.compile(r'\s*PrettyFormat\s+"?(.*)"?')
     CiteFormat = re.compile(r'\s*CiteFormat')
     KeyVal = re.compile(r'^\s*_\w+\s+(.*)$')
+    Float = re.compile(r'\s*Float')
     End = re.compile(r'\s*End')
+    Comment = re.compile(r'\s*#')
+
+    languages = []
+    keyset = set()
+    if layouttranslations:
+        linguas_file = os.path.join(base, 'po/LINGUAS')
+        for line in open(linguas_file).readlines():
+            if Comment.search(line) == None:
+                languages.extend(line.split())
+    # walon is not a known document language
+    # FIXME: Do not hardcode, read from lib/languages!
+    if 'wa' in languages:
+        languages.remove('wa')
 
     for src in input_files:
         readingDescription = False
         readingI18nPreamble = False
+        readingFloat = False
         readingCiteFormats = False
         descStartLine = -1
         descLines = []
@@ -111,6 +128,7 @@ def layouts_l10n(input_files, output, base):
                 if res != None:
                     readingDescription = False
                     desc = " ".join(descLines)
+                    if not layouttranslations:
                         writeString(out, src, base, lineno + 1, desc)
                     continue
                 descLines.append(line[1:].strip())
@@ -128,6 +146,9 @@ def layouts_l10n(input_files, output, base):
                 res = I18nString.search(line)
                 if res != None:
                     string = res.group(1)
+                    if layouttranslations:
+                        keyset.add(string)
+                    else:
                         writeString(out, src, base, lineno, string)
                 continue
             res = I18nPreamble.search(line)
@@ -137,32 +158,44 @@ def layouts_l10n(input_files, output, base):
             res = NameRE.search(line)
             if res != None:
                 string = res.group(1)
+                if not layouttranslations:
                     writeString(out, src, base, lineno + 1, string)
                 continue
             res = Style.search(line)
             if res != None:
                 string = res.group(1)
                 string = string.replace('_', ' ')
+                if not layouttranslations:
                     writeString(out, src, base, lineno, string)
                 continue
             res = LabelString.search(line)
             if res != None:
                 string = res.group(1)
+                if not layouttranslations:
                     writeString(out, src, base, lineno, string)
                 continue
             res = GuiName.search(line)
             if res != None:
                 string = res.group(1)
+                if layouttranslations:
+                    # gui name must only be added for floats
+                    if readingFloat:
+                        keyset.add(string)
+                else:
                     writeString(out, src, base, lineno, string)
                 continue
             res = CategoryName.search(line)
             if res != None:
                 string = res.group(1)
+                if not layouttranslations:
                     writeString(out, src, base, lineno, string)
                 continue
             res = ListName.search(line)
             if res != None:
                 string = res.group(1)
+                if layouttranslations:
+                    keyset.add(string.strip('"'))
+                else:
                     writeString(out, src, base, lineno, string)
                 continue
             res = InsetLayout.search(line)
@@ -170,33 +203,81 @@ def layouts_l10n(input_files, output, base):
                 string = res.group(1)
                 string = string.replace('_', ' ')
                 #Flex:xxx is not used in translation
-                #writeString(out, src, base, lineno, string)
+                #if not layouttranslations:
+                #    writeString(out, src, base, lineno, string)
                 m = FlexCheck.search(string)
                 if m:
+                    if not layouttranslations:
                         writeString(out, src, base, lineno, m.group(1))
                 continue
             res = Category.search(line)
             if res != None:
                 string = res.group(1)
+                if not layouttranslations:
                     writeString(out, src, base, lineno, string)
                 continue
             res = CounterFormat.search(line)
             if res != None:
                 string = res.group(1)
+                if not layouttranslations:
                     writeString(out, src, base, lineno, string)
                 continue
+            res = Float.search(line)
+            if res != None:
+                readingFloat = True
+                continue
             res = CiteFormat.search(line)
             if res != None:
                 readingCiteFormats = True
             res = End.search(line)
-            if res != None and readingCiteFormats:
+            if res != None:
                 readingCiteFormats = False
+                readingFloat = False
             if readingCiteFormats:
                 res = KeyVal.search(line)
                 if res != None:
                     val = res.group(1)
+                    if not layouttranslations:
                         writeString(out, src, base, lineno, val)
 
+    if layouttranslations:
+        # Extract translations of layout files
+        import polib
+
+        # Sort languages and key to minimize the diff between different runs
+        # with changed translations
+        languages.sort()
+        keys = []
+        for key in keyset:
+            keys.append(key)
+        keys.sort()
+
+        print >> out, '''# This file has been automatically generated by po/lyx_pot.py.
+# PLEASE DO NOT MODIFY ANYTHING HERE! If you want to regenerate this file
+# from the translations, run `make ../lib/layouttranslations' in po.'''
+        for lang in languages:
+            print >> out, '\nTranslation %s' % lang
+            poname = os.path.join(base, 'po/' + lang + '.po')
+            po = polib.pofile(poname)
+            # Iterate through po entries and not keys for speed reasons.
+            # FIXME: The code is still too slow
+            trans = dict()
+            for entry in po:
+                if not entry.translated():
+                    continue
+                if entry.msgid in keys:
+                    key = entry.msgid.replace('\\', '\\\\').replace('"', '\\"')
+                    val = entry.msgstr.replace('\\', '\\\\').replace('"', '\\"')
+                    # some translators keep untranslated entries
+                    if val != key:
+                        trans[key] = val
+            for key in keys:
+                if key in trans.keys():
+                    val = trans[key]
+                    print >> out, '\t"%s" "%s"' % \
+                             (key.encode('utf-8'), val.encode('utf-8'))
+            print >> out, 'End'
+
     out.close()
 
 
@@ -364,6 +445,7 @@ where
     --input_type can be
         ui: lib/ui/*
         layouts: lib/layouts/*
+        layouttranslations: create lib/layouttranslations from po/*.po and lib/layouts/*
         qt4: qt4 ui files
         languages: file lib/languages
         encodings: file lib/encodings
@@ -392,7 +474,7 @@ if __name__ == '__main__':
         elif opt in ['-s', '--src_file']:
             input_files = [f.strip() for f in open(value)]
 
-    if input_type not in ['ui', 'layouts', 'modules', 'qt4', 'languages', 'encodings', 'external', 'formats'] or output is None:
+    if input_type not in ['ui', 'layouts', 'layouttranslations', 'qt4', 'languages', 'encodings', 'external', 'formats'] or output is None:
         print 'Wrong input type or output filename.'
         sys.exit(1)
 
@@ -401,7 +483,9 @@ if __name__ == '__main__':
     if input_type == 'ui':
         ui_l10n(input_files, output, base)
     elif input_type == 'layouts':
-        layouts_l10n(input_files, output, base)
+        layouts_l10n(input_files, output, base, False)
+    elif input_type == 'layouttranslations':
+        layouts_l10n(input_files, output, base, True)
     elif input_type == 'qt4':
         qt4_l10n(input_files, output, base)
     elif input_type == 'external':

src/LaTeXFeatures.cpp

@@ -1202,8 +1202,7 @@ docstring const LaTeXFeatures::getTClassI18nPreamble(bool use_babel, bool use_po
 				continue;
 			docstring const type = from_ascii(fl.floattype());
 			docstring const flname = from_utf8(fl.name());
-			docstring name = translateIfPossible(flname,
+			docstring name = buffer().language()->translateLayout(fl.name());
-				buffer().language()->code());
 			// only request translation if we have a real translation
 			// (that differs from the source)
 			if (use_polyglossia && flname != name)

src/Language.cpp

@@ -21,6 +21,7 @@
 
 #include "support/debug.h"
 #include "support/FileName.h"
+#include "support/filetools.h"
 #include "support/lstrings.h"
 #include "support/Messages.h"
 
@@ -39,6 +40,25 @@ Language const * latex_language = &latex_lang;
 Language const * reset_language = 0;
 
 
+docstring const Language::translateLayout(string const & m) const
+{
+	if (m.empty())
+		return docstring();
+
+	if (!isAscii(m)) {
+		lyxerr << "Warning: not translating `" << m
+		       << "' because it is not pure ASCII.\n";
+		return from_utf8(m);
+	}
+
+	TranslationMap::const_iterator it = layoutTranslations_.find(m);
+	if (it != layoutTranslations_.end())
+		return it->second;
+
+	return from_ascii(m);
+}
+
+
 bool Language::readLanguage(Lexer & lex)
 {
 	enum LanguageTags {
@@ -173,6 +193,31 @@ bool Language::read(Lexer & lex)
 	return true;
 }
 
+
+bool Language::readLayoutTranslations(Lexer & lex)
+{
+	layoutTranslations_.clear();
+	while (lex.isOK()) {
+		if (lex.checkFor("End"))
+			break;
+		if (!lex.next(true))
+			return false;
+		string const key = lex.getString();
+		if (!lex.next(true))
+			return false;
+		docstring const val = lex.getDocString();
+		layoutTranslations_[key] = val;
+	}
+	return true;
+}
+
+
+void Language::readLayoutTranslations(Language const & lang)
+{
+	layoutTranslations_ = lang.layoutTranslations_;
+}
+
+
 void Languages::read(FileName const & filename)
 {
 	Lexer lex;
@@ -212,6 +257,61 @@ void Languages::read(FileName const & filename)
 			default_language = &(*languagelist.begin()).second;
 		LYXERR0("Using \"" << default_language->lang() << "\" instead!");
 	}
+
+	// Read layout translations
+	FileName const path = libFileSearch(string(), "layouttranslations");
+	readLayoutTranslations(path);
+}
+
+
+void Languages::readLayoutTranslations(support::FileName const & filename)
+{
+	Lexer lex;
+	lex.setFile(filename);
+	lex.setContext("Languages::read");
+	while (lex.isOK()) {
+		if (!lex.checkFor("Translation")) {
+			if (lex.isOK())
+				lex.printError("Unknown layout translation tag `$$Token'");
+			break;
+		}
+		if (!lex.next(true))
+			break;
+		string const code = lex.getString();
+		// we need to mimic gettext: code can be a two-letter code,
+		// which should match all variants, e.g. "de" should match
+		// "de_DE", "de_AT" etc.
+		Language * firstlang = 0;
+		LanguageList::iterator const end = languagelist.end();
+		for (LanguageList::iterator it = languagelist.begin(); it != end; ++it) {
+			// special case for chinese:
+			// simplified  => code == "zh_CN", langcode == "zh_CN"
+			// traditional => code == "zh_TW", langcode == "zh_CN"
+			string const langcode = it->second.code();
+			string const name = it->second.lang();
+			if ((code == langcode && name != "chinese-traditional") ||
+			    (code == "zh_TW" && name == "chinese-traditional") ||
+			    (code.size() == 2 && langcode.size() > 2 &&
+			     code + '_' == langcode.substr(0, 3))) {
+				if (firstlang)
+					it->second.readLayoutTranslations(*firstlang);
+				else {
+					if (!it->second.readLayoutTranslations(lex)) {
+						lex.printError("Could not read "
+						               "layout translations "
+						               "for language `" +
+						               code + "'");
+						break;
+					}
+					firstlang = &(it->second);
+				}
+			}
+		}
+		if (!firstlang) {
+			lex.printError("Unknown language `" + code + "'");
+			break;
+		}
+	}
 }
 
 

src/Language.h

@@ -15,8 +15,9 @@
 #ifndef LANGUAGE_H
 #define LANGUAGE_H
 
+#include "support/docstring.h"
+
 #include <map>
-#include <string>
 
 
 namespace lyx {
@@ -45,6 +46,13 @@ public:
 	bool rightToLeft() const { return rightToLeft_; }
 	/// Is an (at least partial) translation of this language available?
 	bool translated() const { return translated_; }
+	/**
+	 * Translate a string from the layout files that appears in the output.
+	 * It takes the translations from lib/layouttranslations instead of
+	 * the .mo files. This should be used for every translation that
+	 * appears in the exported document, since the output must not depend
+	 * on installed locales. Non-ASCII keys are not translated. */
+	docstring const translateLayout(std::string const & msg) const;
 	/// default encoding
 	Encoding const * encoding() const { return encoding_; }
 	///
@@ -69,6 +77,10 @@ public:
 	bool read(Lexer & lex);
 	///
 	bool readLanguage(Lexer & lex);
+	///
+	bool readLayoutTranslations(Lexer & lex);
+	///
+	void readLayoutTranslations(Language const & lang);
 	// for the use in std::map
 	friend bool operator<(Language const & p, Language const & q);
 private:
@@ -102,6 +114,10 @@ private:
 	bool as_babel_options_;
 	///
 	bool translated_;
+	///
+	typedef std::map<std::string, docstring> TranslationMap;
+	///
+	TranslationMap layoutTranslations_;
 };
 
 
@@ -123,6 +139,8 @@ public:
 	///
 	void read(support::FileName const & filename);
 	///
+	void readLayoutTranslations(support::FileName const & filename);
+	///
 	Language const * getLanguage(std::string const & language) const;
 	///
 	size_type size() const { return languagelist.size(); }

src/Layout.cpp

@@ -897,14 +897,7 @@ docstring const i18npreamble(Language const * lang, docstring const & templ, boo
 	smatch sub;
 	while (regex_search(preamble, sub, reg)) {
 		string const key = sub.str(1);
-		string translated;
+		string translated = to_utf8(lang->translateLayout(key));
-		if (isAscii(key))
-			translated = to_utf8(getMessages(lang->code()).get(key));
-		else {
-			lyxerr << "Warning: not translating `" << key
-			       << "' because it is not pure ASCII." << endl;
-			translated = key;
-		}
 		preamble = subst(preamble, sub.str(), translated);
 	}
 #endif