From f75b0815510f1beb07c219c42c9c866c4162fae6 Mon Sep 17 00:00:00 2001 From: Juergen Spitzmueller Date: Fri, 29 Mar 2019 15:45:55 +0100 Subject: [PATCH] Add support for mixed-encoded biblatex files Biblatex 3.12 allows to specifiy individual encodings per bib file via \addbibresource[bibencoding=]. This is now supported via GuiBibtex. --- development/FORMAT | 5 ++ lib/lyx2lyx/lyx_2_4.py | 116 ++++++++++++++++++++++++++++++- src/Buffer.cpp | 29 ++++++-- src/Buffer.h | 4 +- src/BufferParams.cpp | 16 +++++ src/BufferParams.h | 11 +-- src/frontends/qt4/GuiBibtex.cpp | 82 ++++++++++++++++++++-- src/frontends/qt4/GuiBibtex.h | 6 ++ src/frontends/qt4/ui/BibtexUi.ui | 6 +- src/insets/InsetBibtex.cpp | 33 ++++++++- src/insets/InsetBibtex.h | 2 + src/version.h | 4 +- 12 files changed, 286 insertions(+), 28 deletions(-) diff --git a/development/FORMAT b/development/FORMAT index 007f9ebf9b..3ec34e61f7 100644 --- a/development/FORMAT +++ b/development/FORMAT @@ -7,6 +7,11 @@ changes happened in particular if possible. A good example would be ----------------------- +2019-03-29 Jürgen Spitzmüller + * format incremented to 570: Add individual bib encodings for biblatex + \begin_inset CommandInset bibtex + file_encodings " \t " + 2019-03-26 Jürgen Spitzmüller * format incremented to 569: New buffer param \tablestyle Determines the standard table template to be used. diff --git a/lib/lyx2lyx/lyx_2_4.py b/lib/lyx2lyx/lyx_2_4.py index 6b43eceb4b..33b2a3984f 100644 --- a/lib/lyx2lyx/lyx_2_4.py +++ b/lib/lyx2lyx/lyx_2_4.py @@ -835,6 +835,9 @@ def revert_bibencoding(document): k = find_token(document.body, "encoding", i, j) if k != -1: del document.body[k] + if encoding == "default": + i += 1 + continue # Re-find inset end line j = find_end_of_inset(document.body, i) if biblatex: @@ -1443,6 +1446,115 @@ def revert_tablestyle(document): del document.header[i] +def revert_bibfileencodings(document): + " Revert individual Biblatex bibliography encodings " + + # Get cite engine + engine = "basic" + i = find_token(document.header, "\\cite_engine", 0) + if i == -1: + document.warning("Malformed document! Missing \\cite_engine") + else: + engine = get_value(document.header, "\\cite_engine", i) + + # Check if biblatex + biblatex = False + if engine in ["biblatex", "biblatex-natbib"]: + biblatex = True + + # Map lyx to latex encoding names + encodings = { + "utf8" : "utf8", + "utf8x" : "utf8x", + "armscii8" : "armscii8", + "iso8859-1" : "latin1", + "iso8859-2" : "latin2", + "iso8859-3" : "latin3", + "iso8859-4" : "latin4", + "iso8859-5" : "iso88595", + "iso8859-6" : "8859-6", + "iso8859-7" : "iso-8859-7", + "iso8859-8" : "8859-8", + "iso8859-9" : "latin5", + "iso8859-13" : "latin7", + "iso8859-15" : "latin9", + "iso8859-16" : "latin10", + "applemac" : "applemac", + "cp437" : "cp437", + "cp437de" : "cp437de", + "cp850" : "cp850", + "cp852" : "cp852", + "cp855" : "cp855", + "cp858" : "cp858", + "cp862" : "cp862", + "cp865" : "cp865", + "cp866" : "cp866", + "cp1250" : "cp1250", + "cp1251" : "cp1251", + "cp1252" : "cp1252", + "cp1255" : "cp1255", + "cp1256" : "cp1256", + "cp1257" : "cp1257", + "koi8-r" : "koi8-r", + "koi8-u" : "koi8-u", + "pt154" : "pt154", + "utf8-platex" : "utf8", + "ascii" : "ascii" + } + + i = 0 + bibresources = [] + while (True): + i = find_token(document.body, "\\begin_inset CommandInset bibtex", i) + if i == -1: + break + j = find_end_of_inset(document.body, i) + if j == -1: + document.warning("Can't find end of bibtex inset at line %d!!" %(i)) + i += 1 + continue + encodings = get_quoted_value(document.body, "file_encodings", i, j) + if not encodings: + i += 1 + continue + bibfiles = get_quoted_value(document.body, "bibfiles", i, j).split(",") + opts = get_quoted_value(document.body, "biblatexopts", i, j) + if len(bibfiles) == 0: + document.warning("Bibtex inset at line %d does not have a bibfile!" %(i)) + # remove encoding line + k = find_token(document.body, "file_encodings", i, j) + if k != -1: + del document.body[k] + # Re-find inset end line + j = find_end_of_inset(document.body, i) + if biblatex: + enclist = encodings.split("\t") + encmap = dict() + for pp in enclist: + ppp = pp.split(" ", 1) + encmap[ppp[0]] = ppp[1] + for bib in bibfiles: + pr = "\\addbibresource" + if bib in encmap.keys(): + pr += "[bibencoding=" + encmap[bib] + "]" + pr += "{" + bib + "}" + add_to_preamble(document, [pr]) + # Insert ERT \\printbibliography and wrap bibtex inset to a Note + pcmd = "printbibliography" + if opts: + pcmd += "[" + opts + "]" + repl = ["\\begin_inset ERT", "status open", "", "\\begin_layout Plain Layout",\ + "", "", "\\backslash", pcmd, "\\end_layout", "", "\\end_inset", "", "",\ + "\\end_layout", "", "\\begin_layout Standard", "\\begin_inset Note Note",\ + "status open", "", "\\begin_layout Plain Layout" ] + repl += document.body[i:j+1] + repl += ["", "\\end_layout", "", "\\end_inset", "", ""] + document.body[i:j+1] = repl + j += 27 + + i = j + 1 + + ## # Conversion hub @@ -1474,10 +1586,12 @@ convert = [ [566, [convert_hebrew_parentheses]], [567, []], [568, []], - [569, []] + [569, []], + [570, []] ] revert = [ + [569, [revert_bibfileencodings]], [568, [revert_tablestyle]], [567, [revert_soul]], [566, [revert_malayalam]], diff --git a/src/Buffer.cpp b/src/Buffer.cpp index 22647351d4..388fbe16e1 100644 --- a/src/Buffer.cpp +++ b/src/Buffer.cpp @@ -1968,10 +1968,14 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os, // Biblatex bibliographies are loaded here if (params().useBiblatex()) { - vector const bibfiles = + vector> const bibfiles = prepareBibFilePaths(runparams, getBibfiles(), true); - for (docstring const & file: bibfiles) - os << "\\addbibresource{" << file << "}\n"; + for (pair const & file: bibfiles) { + os << "\\addbibresource"; + if (!file.second.empty()) + os << "[bibencoding=" << file.second << "]"; + os << "{" << file.first << "}\n"; + } } if (!runparams.dryrun && features.hasPolyglossiaExclusiveLanguages() @@ -3299,7 +3303,7 @@ string const Buffer::prepareFileNameForLaTeX(string const & name, } -vector const Buffer::prepareBibFilePaths(OutputParams const & runparams, +vector> const Buffer::prepareBibFilePaths(OutputParams const & runparams, docstring_list const & bibfilelist, bool const add_extension) const { @@ -3313,7 +3317,7 @@ vector const Buffer::prepareBibFilePaths(OutputParams const & runpara // Otherwise, store the (maybe absolute) path to the original, // unmangled database name. - vector res; + vector> res; // determine the export format string const tex_format = flavor2format(runparams.flavor); @@ -3385,9 +3389,20 @@ vector const Buffer::prepareBibFilePaths(OutputParams const & runpara if (contains(path, ' ')) found_space = true; + string enc; + if (params().useBiblatex() && !params().bibFileEncoding(utf8input).empty()) + enc = params().bibFileEncoding(utf8input); - if (find(res.begin(), res.end(), path) == res.end()) - res.push_back(path); + bool recorded = false; + for (pair pe : res) { + if (pe.first == path) { + recorded = true; + break; + } + + } + if (!recorded) + res.push_back(make_pair(path, enc)); } // Check if there are spaces in the path and warn BibTeX users, if so. diff --git a/src/Buffer.h b/src/Buffer.h index 99cfce1fe1..4b2ac24192 100644 --- a/src/Buffer.h +++ b/src/Buffer.h @@ -413,9 +413,9 @@ public: std::string const &, bool nice) const; /** Returns a vector of bibliography (*.bib) file paths suitable for the - * output in the respective BibTeX/Biblatex macro + * output in the respective BibTeX/Biblatex macro and potential individual encoding */ - std::vector const prepareBibFilePaths(OutputParams const &, + std::vector> const prepareBibFilePaths(OutputParams const &, const docstring_list & bibfilelist, bool const extension = true) const; diff --git a/src/BufferParams.cpp b/src/BufferParams.cpp index de46966f2e..2234ca4b56 100644 --- a/src/BufferParams.cpp +++ b/src/BufferParams.cpp @@ -3550,4 +3550,20 @@ void BufferParams::copyForAdvFR(const BufferParams & bp) setBaseClass(doc_class); } + +void BufferParams::setBibFileEncoding(string const & file, string const & enc) +{ + bib_encodings[file] = enc; +} + + +string const BufferParams::bibFileEncoding(string const & file) const +{ + if (bib_encodings.find(file) == bib_encodings.end()) + return string(); + return bib_encodings.find(file)->second; +} + + + } // namespace lyx diff --git a/src/BufferParams.h b/src/BufferParams.h index aa8385582f..5c80670a8a 100644 --- a/src/BufferParams.h +++ b/src/BufferParams.h @@ -509,6 +509,10 @@ public: void setBibEncoding(std::string const & s) { bib_encoding = s; } /// Get the bib file encoding (for biblatex) std::string const & bibEncoding() const { return bib_encoding; } + /// Set encoding for individual bib file (for biblatex) + void setBibFileEncoding(std::string const & file, std::string const & enc); + /// + std::string const bibFileEncoding(std::string const & file) const; /// options for pdf output PDFOptions & pdfoptions(); @@ -586,11 +590,10 @@ private: CiteEngineType cite_engine_type_; /// the default BibTeX style file for the document std::string biblio_style; - /// The encoding of the bib files, for Biblatex - /// (only one supported currently) - // FIXME: biblatex 3.12 introduces per-file - // encoding options. Update once that's spread. + /// The main encoding of the bib files, for Biblatex std::string bib_encoding; + /// Individual file encodings, for Biblatex + std::map bib_encodings; /// Split bibliography? bool use_bibtopic; /// diff --git a/src/frontends/qt4/GuiBibtex.cpp b/src/frontends/qt4/GuiBibtex.cpp index a70f976804..a6aa95a703 100644 --- a/src/frontends/qt4/GuiBibtex.cpp +++ b/src/frontends/qt4/GuiBibtex.cpp @@ -90,6 +90,7 @@ GuiBibtex::GuiBibtex(GuiView & lv) connect(browseBibPB, SIGNAL(clicked()), this, SLOT(browseBibPressed())); + selected_model_.insertColumns(0, 1); selectionManager = new GuiSelectionManager(this, availableLV, selectedLV, addBibPB, deletePB, upPB, downPB, &available_model_, &selected_model_); connect(selectionManager, SIGNAL(selectionChanged()), @@ -126,15 +127,20 @@ GuiBibtex::GuiBibtex(GuiView & lv) bc().addReadOnly(bibtocCB); bc().addReadOnly(bibEncodingCO); +#if (QT_VERSION < 0x050000) + selectedLV->horizontalHeader()->setResizeMode(QHeaderView::Stretch); +#else + selectedLV->horizontalHeader()->setSectionResizeMode(QHeaderView::Stretch); +#endif + // Always put the default encoding in the first position. bibEncodingCO->addItem(qt_("Document Encoding"), "default"); - QMap encodinglist; for (auto const & encvar : encodings) { if (!encvar.unsafe() && !encvar.guiName().empty()) - encodinglist.insert(qt_(encvar.guiName()), toqstr(encvar.name())); + encodings_.insert(qt_(encvar.guiName()), toqstr(encvar.name())); } - QMap::const_iterator it = encodinglist.constBegin(); - while (it != encodinglist.constEnd()) { + QMap::const_iterator it = encodings_.constBegin(); + while (it != encodings_.constEnd()) { bibEncodingCO->addItem(it.key(), it.value()); ++it; } @@ -200,7 +206,7 @@ void GuiBibtex::browseBstPressed() QString const filen = changeExtension(file, ""); bool present = false; - unsigned int pres = 0; + int pres = 0; for (int i = 0; i != styleCB->count(); ++i) { if (styleCB->itemText(i) == filen) { @@ -251,6 +257,25 @@ void GuiBibtex::clearSelection() void GuiBibtex::setSelectedBibs(QStringList const sl) { selected_model_.clear(); + QStringList headers; + headers << qt_("Database") + << qt_("File Encoding"); + selected_model_.setHorizontalHeaderLabels(headers); + bool const moreencs = usingBiblatex() && sl.count() > 1; + selectedLV->setColumnHidden(1, !moreencs); + selectedLV->verticalHeader()->setVisible(false); + selectedLV->horizontalHeader()->setVisible(moreencs); + if (moreencs) { + bibEncodingLA->setText(qt_("General E&ncoding:")); + bibEncodingCO->setToolTip(qt_("If your bibliography databases use a different " + "encoding than the LyX document, specify it here. " + "If indivivual databases have different encodings, " + "you can set it in the list above.")); + } else { + bibEncodingLA->setText(qt_("E&ncoding:")); + bibEncodingCO->setToolTip(qt_("If your bibliography databases use a different " + "encoding than the LyX document, specify it here")); + } QStringList::const_iterator it = sl.begin(); QStringList::const_iterator end = sl.end(); for (int i = 0; it != end; ++it, ++i) { @@ -260,6 +285,17 @@ void GuiBibtex::setSelectedBibs(QStringList const sl) si->setToolTip(*it); si->setEditable(false); selected_model_.insertRow(i, si); + QComboBox * cb = new QComboBox; + cb->addItem(qt_("General Encoding"), "general"); + cb->addItem(qt_("Document Encoding"), "auto"); + QMap::const_iterator it = encodings_.constBegin(); + while (it != encodings_.constEnd()) { + cb->addItem(it.key(), it.value()); + ++it; + } + cb->setToolTip(qt_("If this bibliography database uses a different " + "encoding than specified below, set it here")); + selectedLV->setIndexWidget(selected_model_.index(i, 1), cb); } } @@ -354,6 +390,8 @@ void GuiBibtex::updateContents() styleCB->clearEditText(); } else biblatexOptsLE->setText(toqstr(params_["biblatexopts"])); + + setFileEncodings(getVectorFromString(params_["file_encodings"], from_ascii("\t"))); } @@ -393,6 +431,9 @@ void GuiBibtex::applyView() params_["btprint"] = qstring_to_ucs4(btPrintCO->itemData(btPrintCO->currentIndex()).toString()); params_["encoding"] = qstring_to_ucs4(bibEncodingCO->itemData(bibEncodingCO->currentIndex()).toString()); + + if (usingBiblatex()) + params_["file_encodings"] = getStringFromVector(getFileEncodings(), from_ascii("\t")); } @@ -449,6 +490,37 @@ QStringList GuiBibtex::bibFiles(bool const extension) const } +vector GuiBibtex::getFileEncodings() +{ + vector res; + for (int i = 0; i != selected_model_.rowCount(); ++i) { + QStandardItem const * key = selected_model_.item(i, 0); + QComboBox * cb = qobject_cast(selectedLV->indexWidget(selected_model_.index(i, 1))); + QString fenc = cb ? cb->itemData(cb->currentIndex()).toString() : QString(); + if (key && !key->text().isEmpty() && !fenc.isEmpty() && fenc != "general") + res.push_back(qstring_to_ucs4(key->text()) + " " + qstring_to_ucs4(fenc)); + } + return res; +} + + +void GuiBibtex::setFileEncodings(vector const m) +{ + for (docstring const & s: m) { + docstring key; + QString enc = toqstr(split(s, key, ' ')); + QModelIndexList qmil = + selected_model_.match(selected_model_.index(0, 0), + Qt::DisplayRole, toqstr(key), 1, + Qt::MatchFlags(Qt::MatchExactly | Qt::MatchWrap)); + if (!qmil.empty()) { + QComboBox * cb = qobject_cast(selectedLV->indexWidget(selected_model_.index(qmil.front().row(), 1))); + cb->setCurrentIndex(cb->findData(enc)); + } + } +} + + void GuiBibtex::rescanBibStyles() const { if (usingBiblatex()) diff --git a/src/frontends/qt4/GuiBibtex.h b/src/frontends/qt4/GuiBibtex.h index 8063694142..9cbd0d61a4 100644 --- a/src/frontends/qt4/GuiBibtex.h +++ b/src/frontends/qt4/GuiBibtex.h @@ -84,6 +84,10 @@ private: QStringList selectedBibs(); /// void setButtons(); + /// + std::vector getFileEncodings(); + /// + void setFileEncodings(std::vector const m); /// bool initialiseParams(std::string const & data); @@ -109,6 +113,8 @@ private: QStringList selected_bibs_; /// contains the search box FancyLineEdit * filter_; + /// + QMap encodings_; }; } // namespace frontend diff --git a/src/frontends/qt4/ui/BibtexUi.ui b/src/frontends/qt4/ui/BibtexUi.ui index 83d545af2c..b66bffde24 100644 --- a/src/frontends/qt4/ui/BibtexUi.ui +++ b/src/frontends/qt4/ui/BibtexUi.ui @@ -169,11 +169,7 @@ - - - QAbstractItemView::NoEditTriggers - - + diff --git a/src/insets/InsetBibtex.cpp b/src/insets/InsetBibtex.cpp index ae76a7f26c..c4bf8721c1 100644 --- a/src/insets/InsetBibtex.cpp +++ b/src/insets/InsetBibtex.cpp @@ -73,6 +73,7 @@ ParamInfo const & InsetBibtex::findInfo(string const & /* cmdName */) param_info_.add("bibfiles", ParamInfo::LATEX_REQUIRED); param_info_.add("options", ParamInfo::LYX_INTERNAL); param_info_.add("encoding", ParamInfo::LYX_INTERNAL); + param_info_.add("file_encodings", ParamInfo::LYX_INTERNAL); param_info_.add("biblatexopts", ParamInfo::LATEX_OPTIONAL); } return param_info_; @@ -290,8 +291,11 @@ void InsetBibtex::latex(otexstream & os, OutputParams const & runparams) const os << "\n"; } else {// using BibTeX // Database(s) - vector const db_out = + vector> const dbs = buffer().prepareBibFilePaths(runparams, getBibFiles(), false); + vector db_out; + for (pair const & db : dbs) + db_out.push_back(db.first); // Style options if (style == "default") style = buffer().masterParams().defaultBiblioStyle(); @@ -895,10 +899,35 @@ void InsetBibtex::updateBuffer(ParIterator const &, UpdateType) // record encoding of bib files for biblatex string const enc = (params()["encoding"] == from_ascii("default")) ? string() : to_ascii(params()["encoding"]); + bool invalidate = false; if (buffer().params().bibEncoding() != enc) { buffer().params().setBibEncoding(enc); - buffer().invalidateBibinfoCache(); + invalidate = true; } + map encs = getFileEncodings(); + map::const_iterator it = encs.begin(); + for (; it != encs.end(); ++it) { + if (buffer().params().bibFileEncoding(it->first) != it->second) { + buffer().params().setBibFileEncoding(it->first, it->second); + invalidate = true; + } + } + if (invalidate) + buffer().invalidateBibinfoCache(); +} + + +map InsetBibtex::getFileEncodings() const +{ + vector ps = + getVectorFromString(to_utf8(getParam("file_encodings")), "\t"); + std::map res; + for (string const & s: ps) { + string key; + string val = split(s, key, ' '); + res[key] = val; + } + return res; } diff --git a/src/insets/InsetBibtex.h b/src/insets/InsetBibtex.h index 63c99f9ad5..4f38d84070 100644 --- a/src/insets/InsetBibtex.h +++ b/src/insets/InsetBibtex.h @@ -89,6 +89,8 @@ private: bool usingBiblatex() const; /// docstring getRefLabel() const; + /// + std::map getFileEncodings() const; /// \name Private functions inherited from Inset class //@{ diff --git a/src/version.h b/src/version.h index dc50c36d94..7783fd75e7 100644 --- a/src/version.h +++ b/src/version.h @@ -32,8 +32,8 @@ extern char const * const lyx_version_info; // Do not remove the comment below, so we get merge conflict in // independent branches. Instead add your own. -#define LYX_FORMAT_LYX 569 // spitz: tablestyle buffer param -#define LYX_FORMAT_TEX2LYX 569 +#define LYX_FORMAT_LYX 570 // spitz: biblatex bibencodings +#define LYX_FORMAT_TEX2LYX 570 #if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX #ifndef _MSC_VER