Add support for mixed-encoded biblatex files

Biblatex 3.12 allows to specifiy individual encodings per bib file
via \addbibresource[bibencoding=<encoding>].

This is now supported via GuiBibtex.

development/FORMAT

@@ -7,6 +7,11 @@ changes happened in particular if possible. A good example would be
 
 -----------------------
 
+2019-03-29  Jürgen Spitzmüller <spitz@lyx.org>
+	* format incremented to 570: Add individual bib encodings for biblatex
+          \begin_inset CommandInset bibtex
+          file_encodings "<bibfile1> <encoding1>\t<bibfile2> <encoding2>"
+
 2019-03-26  Jürgen Spitzmüller <spitz@lyx.org>
 	* format incremented to 569: New buffer param \tablestyle
           Determines the standard table template to be used.

lib/lyx2lyx/lyx_2_4.py

@@ -835,6 +835,9 @@ def revert_bibencoding(document):
         k = find_token(document.body, "encoding", i, j)
         if k != -1:
             del document.body[k]
+        if encoding == "default":
+            i += 1
+            continue
         # Re-find inset end line
         j = find_end_of_inset(document.body, i)
         if biblatex:
@@ -1443,6 +1446,115 @@ def revert_tablestyle(document):
         del document.header[i]
 
 
+def revert_bibfileencodings(document):
+    " Revert individual Biblatex bibliography encodings "
+
+    # Get cite engine
+    engine = "basic"
+    i = find_token(document.header, "\\cite_engine", 0)
+    if i == -1:
+        document.warning("Malformed document! Missing \\cite_engine")
+    else:
+        engine = get_value(document.header, "\\cite_engine", i)
+
+    # Check if biblatex
+    biblatex = False
+    if engine in ["biblatex", "biblatex-natbib"]:
+        biblatex = True
+
+    # Map lyx to latex encoding names
+    encodings = {
+        "utf8" : "utf8",
+        "utf8x" : "utf8x",
+        "armscii8" : "armscii8",
+        "iso8859-1" : "latin1",
+        "iso8859-2" : "latin2",
+        "iso8859-3" : "latin3",
+        "iso8859-4" : "latin4",
+        "iso8859-5" : "iso88595",
+        "iso8859-6" : "8859-6",
+        "iso8859-7" : "iso-8859-7",
+        "iso8859-8" : "8859-8",
+        "iso8859-9" : "latin5",
+        "iso8859-13" : "latin7",
+        "iso8859-15" : "latin9",
+        "iso8859-16" : "latin10",
+        "applemac" : "applemac",
+        "cp437" : "cp437",
+        "cp437de" : "cp437de",
+        "cp850" : "cp850",
+        "cp852" : "cp852",
+        "cp855" : "cp855",
+        "cp858" : "cp858",
+        "cp862" : "cp862",
+        "cp865" : "cp865",
+        "cp866" : "cp866",
+        "cp1250" : "cp1250",
+        "cp1251" : "cp1251",
+        "cp1252" : "cp1252",
+        "cp1255" : "cp1255",
+        "cp1256" : "cp1256",
+        "cp1257" : "cp1257",
+        "koi8-r" : "koi8-r",
+        "koi8-u" : "koi8-u",
+        "pt154" : "pt154",
+        "utf8-platex" : "utf8",
+        "ascii" : "ascii"
+    }
+
+    i = 0
+    bibresources = []
+    while (True):
+        i = find_token(document.body, "\\begin_inset CommandInset bibtex", i)
+        if i == -1:
+            break
+        j = find_end_of_inset(document.body, i)
+        if j == -1:
+            document.warning("Can't find end of bibtex inset at line %d!!" %(i))
+            i += 1
+            continue
+        encodings = get_quoted_value(document.body, "file_encodings", i, j)
+        if not encodings:
+            i += 1
+            continue
+        bibfiles = get_quoted_value(document.body, "bibfiles", i, j).split(",")
+        opts = get_quoted_value(document.body, "biblatexopts", i, j)
+        if len(bibfiles) == 0:
+            document.warning("Bibtex inset at line %d does not have a bibfile!" %(i))
+        # remove encoding line
+        k = find_token(document.body, "file_encodings", i, j)
+        if k != -1:
+            del document.body[k]
+        # Re-find inset end line
+        j = find_end_of_inset(document.body, i)
+        if biblatex:
+            enclist = encodings.split("\t")
+            encmap = dict()
+            for pp in enclist:
+                ppp = pp.split(" ", 1)
+                encmap[ppp[0]] = ppp[1]
+            for bib in bibfiles:
+                pr = "\\addbibresource"
+                if bib in encmap.keys():
+                    pr += "[bibencoding=" + encmap[bib] + "]"
+                pr += "{" + bib + "}"
+                add_to_preamble(document, [pr])
+            # Insert ERT \\printbibliography and wrap bibtex inset to a Note
+            pcmd = "printbibliography"
+            if opts:
+                pcmd += "[" + opts + "]"
+            repl = ["\\begin_inset ERT", "status open", "", "\\begin_layout Plain Layout",\
+                    "", "", "\\backslash", pcmd, "\\end_layout", "", "\\end_inset", "", "",\
+                    "\\end_layout", "", "\\begin_layout Standard", "\\begin_inset Note Note",\
+                    "status open", "", "\\begin_layout Plain Layout" ]
+            repl += document.body[i:j+1]
+            repl += ["", "\\end_layout", "", "\\end_inset", "", ""]
+            document.body[i:j+1] = repl
+            j += 27
+
+        i = j + 1
+
+
 
 ##
 # Conversion hub
@@ -1474,10 +1586,12 @@ convert = [
            [566, [convert_hebrew_parentheses]],
            [567, []],
            [568, []],
-           [569, []]
+           [569, []],
+           [570, []]
           ]
 
 revert =  [
+           [569, [revert_bibfileencodings]],
            [568, [revert_tablestyle]],
            [567, [revert_soul]],
            [566, [revert_malayalam]],

src/Buffer.cpp

@@ -1968,10 +1968,14 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
 
 		// Biblatex bibliographies are loaded here
 		if (params().useBiblatex()) {
-			vector<docstring> const bibfiles =
+			vector<pair<docstring, string>> const bibfiles =
 				prepareBibFilePaths(runparams, getBibfiles(), true);
-			for (docstring const & file: bibfiles)
+			for (pair<docstring, string> const & file: bibfiles) {
-				os << "\\addbibresource{" << file << "}\n";
+				os << "\\addbibresource";
+				if (!file.second.empty())
+					os << "[bibencoding=" << file.second << "]";
+				os << "{" << file.first << "}\n";
+			}
 		}
 
 		if (!runparams.dryrun && features.hasPolyglossiaExclusiveLanguages()
@@ -3299,7 +3303,7 @@ string const Buffer::prepareFileNameForLaTeX(string const & name,
 }
 
 
-vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
+vector<pair<docstring, string>> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
 						docstring_list const & bibfilelist,
 						bool const add_extension) const
 {
@@ -3313,7 +3317,7 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
 	// Otherwise, store the (maybe absolute) path to the original,
 	// unmangled database name.
 
-	vector<docstring> res;
+	vector<pair<docstring, string>> res;
 
 	// determine the export format
 	string const tex_format = flavor2format(runparams.flavor);
@@ -3385,9 +3389,20 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
 
 		if (contains(path, ' '))
 			found_space = true;
+		string enc;
+		if (params().useBiblatex() && !params().bibFileEncoding(utf8input).empty())
+			enc = params().bibFileEncoding(utf8input);
 
-		if (find(res.begin(), res.end(), path) == res.end())
+		bool recorded = false;
-			res.push_back(path);
+		for (pair<docstring, string> pe : res) {
+			if (pe.first == path) {
+				recorded = true;
+				break;
+			}
+
+		}
+		if (!recorded)
+			res.push_back(make_pair(path, enc));
 	}
 
 	// Check if there are spaces in the path and warn BibTeX users, if so.

src/Buffer.h

@@ -413,9 +413,9 @@ public:
 					std::string const &, bool nice) const;
 
 	/** Returns a vector of bibliography (*.bib) file paths suitable for the
-	 *  output in the respective BibTeX/Biblatex macro
+	 *  output in the respective BibTeX/Biblatex macro and potential individual encoding
 	 */
-	std::vector<docstring> const prepareBibFilePaths(OutputParams const &,
+	std::vector<std::pair<docstring, std::string>> const prepareBibFilePaths(OutputParams const &,
 				    const docstring_list & bibfilelist,
 				    bool const extension = true) const;
 

src/BufferParams.cpp

@@ -3550,4 +3550,20 @@ void BufferParams::copyForAdvFR(const BufferParams & bp)
 	setBaseClass(doc_class);
 }
 
+
+void BufferParams::setBibFileEncoding(string const & file, string const & enc)
+{
+	bib_encodings[file] = enc;
+}
+
+
+string const BufferParams::bibFileEncoding(string const & file) const
+{
+	if (bib_encodings.find(file) == bib_encodings.end())
+		return string();
+	return bib_encodings.find(file)->second;
+}
+
+
+
 } // namespace lyx

src/BufferParams.h

@@ -509,6 +509,10 @@ public:
 	void setBibEncoding(std::string const & s) { bib_encoding = s; }
 	/// Get the bib file encoding (for biblatex)
 	std::string const & bibEncoding() const { return bib_encoding; }
+	/// Set encoding for individual bib file (for biblatex)
+	void setBibFileEncoding(std::string const & file, std::string const & enc);
+	///
+	std::string const bibFileEncoding(std::string const & file) const;
 
 	/// options for pdf output
 	PDFOptions & pdfoptions();
@@ -586,11 +590,10 @@ private:
 	CiteEngineType cite_engine_type_;
 	/// the default BibTeX style file for the document
 	std::string biblio_style;
-	/// The encoding of the bib files, for Biblatex
+	/// The main encoding of the bib files, for Biblatex
-	/// (only one supported currently)
-	// FIXME: biblatex 3.12 introduces per-file
-	// encoding options. Update once that's spread.
 	std::string bib_encoding;
+	/// Individual file encodings, for Biblatex
+	std::map<std::string, std::string> bib_encodings;
 	/// Split bibliography?
 	bool use_bibtopic;
 	///

src/frontends/qt4/GuiBibtex.cpp

@@ -90,6 +90,7 @@ GuiBibtex::GuiBibtex(GuiView & lv)
 	connect(browseBibPB, SIGNAL(clicked()),
 		this, SLOT(browseBibPressed()));
 
+	selected_model_.insertColumns(0, 1);
 	selectionManager = new GuiSelectionManager(this, availableLV, selectedLV,
 			addBibPB, deletePB, upPB, downPB, &available_model_, &selected_model_);
 	connect(selectionManager, SIGNAL(selectionChanged()),
@@ -126,15 +127,20 @@ GuiBibtex::GuiBibtex(GuiView & lv)
 	bc().addReadOnly(bibtocCB);
 	bc().addReadOnly(bibEncodingCO);
 
+#if (QT_VERSION < 0x050000)
+	selectedLV->horizontalHeader()->setResizeMode(QHeaderView::Stretch);
+#else
+	selectedLV->horizontalHeader()->setSectionResizeMode(QHeaderView::Stretch);
+#endif
+
 	// Always put the default encoding in the first position.
 	bibEncodingCO->addItem(qt_("Document Encoding"), "default");
-	QMap<QString, QString> encodinglist;
 	for (auto const & encvar : encodings) {
 		if (!encvar.unsafe() && !encvar.guiName().empty())
-			encodinglist.insert(qt_(encvar.guiName()), toqstr(encvar.name()));
+			encodings_.insert(qt_(encvar.guiName()), toqstr(encvar.name()));
 	}
-	QMap<QString, QString>::const_iterator it = encodinglist.constBegin();
+	QMap<QString, QString>::const_iterator it = encodings_.constBegin();
-	while (it != encodinglist.constEnd()) {
+	while (it != encodings_.constEnd()) {
 		bibEncodingCO->addItem(it.key(), it.value());
 		++it;
 	}
@@ -200,7 +206,7 @@ void GuiBibtex::browseBstPressed()
 
 	QString const filen = changeExtension(file, "");
 	bool present = false;
-	unsigned int pres = 0;
+	int pres = 0;
 
 	for (int i = 0; i != styleCB->count(); ++i) {
 		if (styleCB->itemText(i) == filen) {
@@ -251,6 +257,25 @@ void GuiBibtex::clearSelection()
 void GuiBibtex::setSelectedBibs(QStringList const sl)
 {
 	selected_model_.clear();
+	QStringList headers;
+	headers << qt_("Database")
+		<< qt_("File Encoding");
+	selected_model_.setHorizontalHeaderLabels(headers);
+	bool const moreencs = usingBiblatex() && sl.count() > 1;
+	selectedLV->setColumnHidden(1, !moreencs);
+	selectedLV->verticalHeader()->setVisible(false);
+	selectedLV->horizontalHeader()->setVisible(moreencs);
+	if (moreencs) {
+		bibEncodingLA->setText(qt_("General E&ncoding:"));
+		bibEncodingCO->setToolTip(qt_("If your bibliography databases use a different "
+					      "encoding than the LyX document, specify it here. "
+					      "If indivivual databases have different encodings, "
+					      "you can set it in the list above."));
+	} else {
+		bibEncodingLA->setText(qt_("E&ncoding:"));
+		bibEncodingCO->setToolTip(qt_("If your bibliography databases use a different "
+					      "encoding than the LyX document, specify it here"));
+	}
 	QStringList::const_iterator it  = sl.begin();
 	QStringList::const_iterator end = sl.end();
 	for (int i = 0; it != end; ++it, ++i) {
@@ -260,6 +285,17 @@ void GuiBibtex::setSelectedBibs(QStringList const sl)
 		si->setToolTip(*it);
 		si->setEditable(false);
 		selected_model_.insertRow(i, si);
+		QComboBox * cb = new QComboBox;
+		cb->addItem(qt_("General Encoding"), "general");
+		cb->addItem(qt_("Document Encoding"), "auto");
+		QMap<QString, QString>::const_iterator it = encodings_.constBegin();
+		while (it != encodings_.constEnd()) {
+			cb->addItem(it.key(), it.value());
+			++it;
+		}
+		cb->setToolTip(qt_("If this bibliography database uses a different "
+				   "encoding than specified below, set it here"));
+		selectedLV->setIndexWidget(selected_model_.index(i, 1), cb);
 	}
 }
 
@@ -354,6 +390,8 @@ void GuiBibtex::updateContents()
 			styleCB->clearEditText();
 	} else
 		biblatexOptsLE->setText(toqstr(params_["biblatexopts"]));
+
+	setFileEncodings(getVectorFromString(params_["file_encodings"], from_ascii("\t")));
 }
 
 
@@ -393,6 +431,9 @@ void GuiBibtex::applyView()
 	params_["btprint"] = qstring_to_ucs4(btPrintCO->itemData(btPrintCO->currentIndex()).toString());
 
 	params_["encoding"] = qstring_to_ucs4(bibEncodingCO->itemData(bibEncodingCO->currentIndex()).toString());
+
+	if (usingBiblatex())
+		params_["file_encodings"] = getStringFromVector(getFileEncodings(), from_ascii("\t"));
 }
 
 
@@ -449,6 +490,37 @@ QStringList GuiBibtex::bibFiles(bool const extension) const
 }
 
 
+vector<docstring> GuiBibtex::getFileEncodings()
+{
+	vector<docstring> res;
+	for (int i = 0; i != selected_model_.rowCount(); ++i) {
+		QStandardItem const * key = selected_model_.item(i, 0);
+		QComboBox * cb = qobject_cast<QComboBox*>(selectedLV->indexWidget(selected_model_.index(i, 1)));
+		QString fenc = cb ? cb->itemData(cb->currentIndex()).toString() : QString();
+		if (key && !key->text().isEmpty() && !fenc.isEmpty() && fenc != "general")
+			res.push_back(qstring_to_ucs4(key->text()) + " " + qstring_to_ucs4(fenc));
+	}
+	return res;
+}
+
+
+void GuiBibtex::setFileEncodings(vector<docstring> const m)
+{
+	for (docstring const & s: m) {
+		docstring key;
+		QString enc = toqstr(split(s, key, ' '));
+		QModelIndexList qmil =
+				selected_model_.match(selected_model_.index(0, 0),
+						     Qt::DisplayRole, toqstr(key), 1,
+						     Qt::MatchFlags(Qt::MatchExactly | Qt::MatchWrap));
+		if (!qmil.empty()) {
+			QComboBox * cb = qobject_cast<QComboBox*>(selectedLV->indexWidget(selected_model_.index(qmil.front().row(), 1)));
+			cb->setCurrentIndex(cb->findData(enc));
+		}
+	}
+}
+
+
 void GuiBibtex::rescanBibStyles() const
 {
 	if (usingBiblatex())

src/frontends/qt4/GuiBibtex.h

@@ -84,6 +84,10 @@ private:
 	QStringList selectedBibs();
 	///
 	void setButtons();
+	///
+	std::vector<docstring> getFileEncodings();
+	///
+	void setFileEncodings(std::vector<docstring> const m);
 
 	///
 	bool initialiseParams(std::string const & data);
@@ -109,6 +113,8 @@ private:
 	QStringList selected_bibs_;
 	/// contains the search box
 	FancyLineEdit * filter_;
+	///
+	QMap<QString, QString> encodings_;
 };
 
 } // namespace frontend

src/frontends/qt4/ui/BibtexUi.ui

@@ -169,11 +169,7 @@
          </widget>
         </item>
         <item>
-         <widget class="QListView" name="selectedLV">
+         <widget class="QTableView" name="selectedLV"/>
-          <property name="editTriggers">
-           <set>QAbstractItemView::NoEditTriggers</set>
-          </property>
-         </widget>
         </item>
        </layout>
       </item>

src/insets/InsetBibtex.cpp

@@ -73,6 +73,7 @@ ParamInfo const & InsetBibtex::findInfo(string const & /* cmdName */)
 		param_info_.add("bibfiles", ParamInfo::LATEX_REQUIRED);
 		param_info_.add("options", ParamInfo::LYX_INTERNAL);
 		param_info_.add("encoding", ParamInfo::LYX_INTERNAL);
+		param_info_.add("file_encodings", ParamInfo::LYX_INTERNAL);
 		param_info_.add("biblatexopts", ParamInfo::LATEX_OPTIONAL);
 	}
 	return param_info_;
@@ -290,8 +291,11 @@ void InsetBibtex::latex(otexstream & os, OutputParams const & runparams) const
 		os << "\n";
 	} else {// using BibTeX
 		// Database(s)
-		vector<docstring> const db_out =
+		vector<pair<docstring, string>> const dbs =
 			buffer().prepareBibFilePaths(runparams, getBibFiles(), false);
+		vector<docstring> db_out;
+		for (pair<docstring, string> const & db : dbs)
+			db_out.push_back(db.first);
 		// Style options
 		if (style == "default")
 			style = buffer().masterParams().defaultBiblioStyle();
@@ -895,10 +899,35 @@ void InsetBibtex::updateBuffer(ParIterator const &, UpdateType)
 	// record encoding of bib files for biblatex
 	string const enc = (params()["encoding"] == from_ascii("default")) ?
 				string() : to_ascii(params()["encoding"]);
+	bool invalidate = false;
 	if (buffer().params().bibEncoding() != enc) {
 		buffer().params().setBibEncoding(enc);
-		buffer().invalidateBibinfoCache();
+		invalidate = true;
 	}
+	map<string, string> encs = getFileEncodings();
+	map<string, string>::const_iterator it = encs.begin();
+	for (; it != encs.end(); ++it) {
+		if (buffer().params().bibFileEncoding(it->first) != it->second) {
+			buffer().params().setBibFileEncoding(it->first, it->second);
+			invalidate = true;
+		}
+	}
+	if (invalidate)
+		buffer().invalidateBibinfoCache();
+}
+
+
+map<string, string> InsetBibtex::getFileEncodings() const
+{
+	vector<string> ps =
+		getVectorFromString(to_utf8(getParam("file_encodings")), "\t");
+	std::map<string, string> res;
+	for (string const & s: ps) {
+		string key;
+		string val = split(s, key, ' ');
+		res[key] = val;
+	}
+	return res;
 }
 
 

src/insets/InsetBibtex.h

@@ -89,6 +89,8 @@ private:
 	bool usingBiblatex() const;
 	///
 	docstring getRefLabel() const;
+	///
+	std::map<std::string, std::string> getFileEncodings() const;
 
 	/// \name Private functions inherited from Inset class
 	//@{

src/version.h

@@ -32,8 +32,8 @@ extern char const * const lyx_version_info;
 
 // Do not remove the comment below, so we get merge conflict in
 // independent branches. Instead add your own.
-#define LYX_FORMAT_LYX 569 // spitz: tablestyle buffer param
+#define LYX_FORMAT_LYX 570 // spitz: biblatex bibencodings
-#define LYX_FORMAT_TEX2LYX 569
+#define LYX_FORMAT_TEX2LYX 570
 
 #if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX
 #ifndef _MSC_VER