Add support for mixed-encoded biblatex files

Biblatex 3.12 allows to specifiy individual encodings per bib file
via \addbibresource[bibencoding=<encoding>].

This is now supported via GuiBibtex.
This commit is contained in:
Juergen Spitzmueller 2019-03-29 15:45:55 +01:00
parent 3acb0d8082
commit f75b081551
12 changed files with 286 additions and 28 deletions

View File

@ -7,6 +7,11 @@ changes happened in particular if possible. A good example would be
-----------------------
2019-03-29 Jürgen Spitzmüller <spitz@lyx.org>
* format incremented to 570: Add individual bib encodings for biblatex
\begin_inset CommandInset bibtex
file_encodings "<bibfile1> <encoding1>\t<bibfile2> <encoding2>"
2019-03-26 Jürgen Spitzmüller <spitz@lyx.org>
* format incremented to 569: New buffer param \tablestyle
Determines the standard table template to be used.

View File

@ -835,6 +835,9 @@ def revert_bibencoding(document):
k = find_token(document.body, "encoding", i, j)
if k != -1:
del document.body[k]
if encoding == "default":
i += 1
continue
# Re-find inset end line
j = find_end_of_inset(document.body, i)
if biblatex:
@ -1443,6 +1446,115 @@ def revert_tablestyle(document):
del document.header[i]
def revert_bibfileencodings(document):
" Revert individual Biblatex bibliography encodings "
# Get cite engine
engine = "basic"
i = find_token(document.header, "\\cite_engine", 0)
if i == -1:
document.warning("Malformed document! Missing \\cite_engine")
else:
engine = get_value(document.header, "\\cite_engine", i)
# Check if biblatex
biblatex = False
if engine in ["biblatex", "biblatex-natbib"]:
biblatex = True
# Map lyx to latex encoding names
encodings = {
"utf8" : "utf8",
"utf8x" : "utf8x",
"armscii8" : "armscii8",
"iso8859-1" : "latin1",
"iso8859-2" : "latin2",
"iso8859-3" : "latin3",
"iso8859-4" : "latin4",
"iso8859-5" : "iso88595",
"iso8859-6" : "8859-6",
"iso8859-7" : "iso-8859-7",
"iso8859-8" : "8859-8",
"iso8859-9" : "latin5",
"iso8859-13" : "latin7",
"iso8859-15" : "latin9",
"iso8859-16" : "latin10",
"applemac" : "applemac",
"cp437" : "cp437",
"cp437de" : "cp437de",
"cp850" : "cp850",
"cp852" : "cp852",
"cp855" : "cp855",
"cp858" : "cp858",
"cp862" : "cp862",
"cp865" : "cp865",
"cp866" : "cp866",
"cp1250" : "cp1250",
"cp1251" : "cp1251",
"cp1252" : "cp1252",
"cp1255" : "cp1255",
"cp1256" : "cp1256",
"cp1257" : "cp1257",
"koi8-r" : "koi8-r",
"koi8-u" : "koi8-u",
"pt154" : "pt154",
"utf8-platex" : "utf8",
"ascii" : "ascii"
}
i = 0
bibresources = []
while (True):
i = find_token(document.body, "\\begin_inset CommandInset bibtex", i)
if i == -1:
break
j = find_end_of_inset(document.body, i)
if j == -1:
document.warning("Can't find end of bibtex inset at line %d!!" %(i))
i += 1
continue
encodings = get_quoted_value(document.body, "file_encodings", i, j)
if not encodings:
i += 1
continue
bibfiles = get_quoted_value(document.body, "bibfiles", i, j).split(",")
opts = get_quoted_value(document.body, "biblatexopts", i, j)
if len(bibfiles) == 0:
document.warning("Bibtex inset at line %d does not have a bibfile!" %(i))
# remove encoding line
k = find_token(document.body, "file_encodings", i, j)
if k != -1:
del document.body[k]
# Re-find inset end line
j = find_end_of_inset(document.body, i)
if biblatex:
enclist = encodings.split("\t")
encmap = dict()
for pp in enclist:
ppp = pp.split(" ", 1)
encmap[ppp[0]] = ppp[1]
for bib in bibfiles:
pr = "\\addbibresource"
if bib in encmap.keys():
pr += "[bibencoding=" + encmap[bib] + "]"
pr += "{" + bib + "}"
add_to_preamble(document, [pr])
# Insert ERT \\printbibliography and wrap bibtex inset to a Note
pcmd = "printbibliography"
if opts:
pcmd += "[" + opts + "]"
repl = ["\\begin_inset ERT", "status open", "", "\\begin_layout Plain Layout",\
"", "", "\\backslash", pcmd, "\\end_layout", "", "\\end_inset", "", "",\
"\\end_layout", "", "\\begin_layout Standard", "\\begin_inset Note Note",\
"status open", "", "\\begin_layout Plain Layout" ]
repl += document.body[i:j+1]
repl += ["", "\\end_layout", "", "\\end_inset", "", ""]
document.body[i:j+1] = repl
j += 27
i = j + 1
##
# Conversion hub
@ -1474,10 +1586,12 @@ convert = [
[566, [convert_hebrew_parentheses]],
[567, []],
[568, []],
[569, []]
[569, []],
[570, []]
]
revert = [
[569, [revert_bibfileencodings]],
[568, [revert_tablestyle]],
[567, [revert_soul]],
[566, [revert_malayalam]],

View File

@ -1968,10 +1968,14 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
// Biblatex bibliographies are loaded here
if (params().useBiblatex()) {
vector<docstring> const bibfiles =
vector<pair<docstring, string>> const bibfiles =
prepareBibFilePaths(runparams, getBibfiles(), true);
for (docstring const & file: bibfiles)
os << "\\addbibresource{" << file << "}\n";
for (pair<docstring, string> const & file: bibfiles) {
os << "\\addbibresource";
if (!file.second.empty())
os << "[bibencoding=" << file.second << "]";
os << "{" << file.first << "}\n";
}
}
if (!runparams.dryrun && features.hasPolyglossiaExclusiveLanguages()
@ -3299,7 +3303,7 @@ string const Buffer::prepareFileNameForLaTeX(string const & name,
}
vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
vector<pair<docstring, string>> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
docstring_list const & bibfilelist,
bool const add_extension) const
{
@ -3313,7 +3317,7 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
// Otherwise, store the (maybe absolute) path to the original,
// unmangled database name.
vector<docstring> res;
vector<pair<docstring, string>> res;
// determine the export format
string const tex_format = flavor2format(runparams.flavor);
@ -3385,9 +3389,20 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
if (contains(path, ' '))
found_space = true;
string enc;
if (params().useBiblatex() && !params().bibFileEncoding(utf8input).empty())
enc = params().bibFileEncoding(utf8input);
if (find(res.begin(), res.end(), path) == res.end())
res.push_back(path);
bool recorded = false;
for (pair<docstring, string> pe : res) {
if (pe.first == path) {
recorded = true;
break;
}
}
if (!recorded)
res.push_back(make_pair(path, enc));
}
// Check if there are spaces in the path and warn BibTeX users, if so.

View File

@ -413,9 +413,9 @@ public:
std::string const &, bool nice) const;
/** Returns a vector of bibliography (*.bib) file paths suitable for the
* output in the respective BibTeX/Biblatex macro
* output in the respective BibTeX/Biblatex macro and potential individual encoding
*/
std::vector<docstring> const prepareBibFilePaths(OutputParams const &,
std::vector<std::pair<docstring, std::string>> const prepareBibFilePaths(OutputParams const &,
const docstring_list & bibfilelist,
bool const extension = true) const;

View File

@ -3550,4 +3550,20 @@ void BufferParams::copyForAdvFR(const BufferParams & bp)
setBaseClass(doc_class);
}
void BufferParams::setBibFileEncoding(string const & file, string const & enc)
{
bib_encodings[file] = enc;
}
string const BufferParams::bibFileEncoding(string const & file) const
{
if (bib_encodings.find(file) == bib_encodings.end())
return string();
return bib_encodings.find(file)->second;
}
} // namespace lyx

View File

@ -509,6 +509,10 @@ public:
void setBibEncoding(std::string const & s) { bib_encoding = s; }
/// Get the bib file encoding (for biblatex)
std::string const & bibEncoding() const { return bib_encoding; }
/// Set encoding for individual bib file (for biblatex)
void setBibFileEncoding(std::string const & file, std::string const & enc);
///
std::string const bibFileEncoding(std::string const & file) const;
/// options for pdf output
PDFOptions & pdfoptions();
@ -586,11 +590,10 @@ private:
CiteEngineType cite_engine_type_;
/// the default BibTeX style file for the document
std::string biblio_style;
/// The encoding of the bib files, for Biblatex
/// (only one supported currently)
// FIXME: biblatex 3.12 introduces per-file
// encoding options. Update once that's spread.
/// The main encoding of the bib files, for Biblatex
std::string bib_encoding;
/// Individual file encodings, for Biblatex
std::map<std::string, std::string> bib_encodings;
/// Split bibliography?
bool use_bibtopic;
///

View File

@ -90,6 +90,7 @@ GuiBibtex::GuiBibtex(GuiView & lv)
connect(browseBibPB, SIGNAL(clicked()),
this, SLOT(browseBibPressed()));
selected_model_.insertColumns(0, 1);
selectionManager = new GuiSelectionManager(this, availableLV, selectedLV,
addBibPB, deletePB, upPB, downPB, &available_model_, &selected_model_);
connect(selectionManager, SIGNAL(selectionChanged()),
@ -126,15 +127,20 @@ GuiBibtex::GuiBibtex(GuiView & lv)
bc().addReadOnly(bibtocCB);
bc().addReadOnly(bibEncodingCO);
#if (QT_VERSION < 0x050000)
selectedLV->horizontalHeader()->setResizeMode(QHeaderView::Stretch);
#else
selectedLV->horizontalHeader()->setSectionResizeMode(QHeaderView::Stretch);
#endif
// Always put the default encoding in the first position.
bibEncodingCO->addItem(qt_("Document Encoding"), "default");
QMap<QString, QString> encodinglist;
for (auto const & encvar : encodings) {
if (!encvar.unsafe() && !encvar.guiName().empty())
encodinglist.insert(qt_(encvar.guiName()), toqstr(encvar.name()));
encodings_.insert(qt_(encvar.guiName()), toqstr(encvar.name()));
}
QMap<QString, QString>::const_iterator it = encodinglist.constBegin();
while (it != encodinglist.constEnd()) {
QMap<QString, QString>::const_iterator it = encodings_.constBegin();
while (it != encodings_.constEnd()) {
bibEncodingCO->addItem(it.key(), it.value());
++it;
}
@ -200,7 +206,7 @@ void GuiBibtex::browseBstPressed()
QString const filen = changeExtension(file, "");
bool present = false;
unsigned int pres = 0;
int pres = 0;
for (int i = 0; i != styleCB->count(); ++i) {
if (styleCB->itemText(i) == filen) {
@ -251,6 +257,25 @@ void GuiBibtex::clearSelection()
void GuiBibtex::setSelectedBibs(QStringList const sl)
{
selected_model_.clear();
QStringList headers;
headers << qt_("Database")
<< qt_("File Encoding");
selected_model_.setHorizontalHeaderLabels(headers);
bool const moreencs = usingBiblatex() && sl.count() > 1;
selectedLV->setColumnHidden(1, !moreencs);
selectedLV->verticalHeader()->setVisible(false);
selectedLV->horizontalHeader()->setVisible(moreencs);
if (moreencs) {
bibEncodingLA->setText(qt_("General E&ncoding:"));
bibEncodingCO->setToolTip(qt_("If your bibliography databases use a different "
"encoding than the LyX document, specify it here. "
"If indivivual databases have different encodings, "
"you can set it in the list above."));
} else {
bibEncodingLA->setText(qt_("E&ncoding:"));
bibEncodingCO->setToolTip(qt_("If your bibliography databases use a different "
"encoding than the LyX document, specify it here"));
}
QStringList::const_iterator it = sl.begin();
QStringList::const_iterator end = sl.end();
for (int i = 0; it != end; ++it, ++i) {
@ -260,6 +285,17 @@ void GuiBibtex::setSelectedBibs(QStringList const sl)
si->setToolTip(*it);
si->setEditable(false);
selected_model_.insertRow(i, si);
QComboBox * cb = new QComboBox;
cb->addItem(qt_("General Encoding"), "general");
cb->addItem(qt_("Document Encoding"), "auto");
QMap<QString, QString>::const_iterator it = encodings_.constBegin();
while (it != encodings_.constEnd()) {
cb->addItem(it.key(), it.value());
++it;
}
cb->setToolTip(qt_("If this bibliography database uses a different "
"encoding than specified below, set it here"));
selectedLV->setIndexWidget(selected_model_.index(i, 1), cb);
}
}
@ -354,6 +390,8 @@ void GuiBibtex::updateContents()
styleCB->clearEditText();
} else
biblatexOptsLE->setText(toqstr(params_["biblatexopts"]));
setFileEncodings(getVectorFromString(params_["file_encodings"], from_ascii("\t")));
}
@ -393,6 +431,9 @@ void GuiBibtex::applyView()
params_["btprint"] = qstring_to_ucs4(btPrintCO->itemData(btPrintCO->currentIndex()).toString());
params_["encoding"] = qstring_to_ucs4(bibEncodingCO->itemData(bibEncodingCO->currentIndex()).toString());
if (usingBiblatex())
params_["file_encodings"] = getStringFromVector(getFileEncodings(), from_ascii("\t"));
}
@ -449,6 +490,37 @@ QStringList GuiBibtex::bibFiles(bool const extension) const
}
vector<docstring> GuiBibtex::getFileEncodings()
{
vector<docstring> res;
for (int i = 0; i != selected_model_.rowCount(); ++i) {
QStandardItem const * key = selected_model_.item(i, 0);
QComboBox * cb = qobject_cast<QComboBox*>(selectedLV->indexWidget(selected_model_.index(i, 1)));
QString fenc = cb ? cb->itemData(cb->currentIndex()).toString() : QString();
if (key && !key->text().isEmpty() && !fenc.isEmpty() && fenc != "general")
res.push_back(qstring_to_ucs4(key->text()) + " " + qstring_to_ucs4(fenc));
}
return res;
}
void GuiBibtex::setFileEncodings(vector<docstring> const m)
{
for (docstring const & s: m) {
docstring key;
QString enc = toqstr(split(s, key, ' '));
QModelIndexList qmil =
selected_model_.match(selected_model_.index(0, 0),
Qt::DisplayRole, toqstr(key), 1,
Qt::MatchFlags(Qt::MatchExactly | Qt::MatchWrap));
if (!qmil.empty()) {
QComboBox * cb = qobject_cast<QComboBox*>(selectedLV->indexWidget(selected_model_.index(qmil.front().row(), 1)));
cb->setCurrentIndex(cb->findData(enc));
}
}
}
void GuiBibtex::rescanBibStyles() const
{
if (usingBiblatex())

View File

@ -84,6 +84,10 @@ private:
QStringList selectedBibs();
///
void setButtons();
///
std::vector<docstring> getFileEncodings();
///
void setFileEncodings(std::vector<docstring> const m);
///
bool initialiseParams(std::string const & data);
@ -109,6 +113,8 @@ private:
QStringList selected_bibs_;
/// contains the search box
FancyLineEdit * filter_;
///
QMap<QString, QString> encodings_;
};
} // namespace frontend

View File

@ -169,11 +169,7 @@
</widget>
</item>
<item>
<widget class="QListView" name="selectedLV">
<property name="editTriggers">
<set>QAbstractItemView::NoEditTriggers</set>
</property>
</widget>
<widget class="QTableView" name="selectedLV"/>
</item>
</layout>
</item>

View File

@ -73,6 +73,7 @@ ParamInfo const & InsetBibtex::findInfo(string const & /* cmdName */)
param_info_.add("bibfiles", ParamInfo::LATEX_REQUIRED);
param_info_.add("options", ParamInfo::LYX_INTERNAL);
param_info_.add("encoding", ParamInfo::LYX_INTERNAL);
param_info_.add("file_encodings", ParamInfo::LYX_INTERNAL);
param_info_.add("biblatexopts", ParamInfo::LATEX_OPTIONAL);
}
return param_info_;
@ -290,8 +291,11 @@ void InsetBibtex::latex(otexstream & os, OutputParams const & runparams) const
os << "\n";
} else {// using BibTeX
// Database(s)
vector<docstring> const db_out =
vector<pair<docstring, string>> const dbs =
buffer().prepareBibFilePaths(runparams, getBibFiles(), false);
vector<docstring> db_out;
for (pair<docstring, string> const & db : dbs)
db_out.push_back(db.first);
// Style options
if (style == "default")
style = buffer().masterParams().defaultBiblioStyle();
@ -895,10 +899,35 @@ void InsetBibtex::updateBuffer(ParIterator const &, UpdateType)
// record encoding of bib files for biblatex
string const enc = (params()["encoding"] == from_ascii("default")) ?
string() : to_ascii(params()["encoding"]);
bool invalidate = false;
if (buffer().params().bibEncoding() != enc) {
buffer().params().setBibEncoding(enc);
buffer().invalidateBibinfoCache();
invalidate = true;
}
map<string, string> encs = getFileEncodings();
map<string, string>::const_iterator it = encs.begin();
for (; it != encs.end(); ++it) {
if (buffer().params().bibFileEncoding(it->first) != it->second) {
buffer().params().setBibFileEncoding(it->first, it->second);
invalidate = true;
}
}
if (invalidate)
buffer().invalidateBibinfoCache();
}
map<string, string> InsetBibtex::getFileEncodings() const
{
vector<string> ps =
getVectorFromString(to_utf8(getParam("file_encodings")), "\t");
std::map<string, string> res;
for (string const & s: ps) {
string key;
string val = split(s, key, ' ');
res[key] = val;
}
return res;
}

View File

@ -89,6 +89,8 @@ private:
bool usingBiblatex() const;
///
docstring getRefLabel() const;
///
std::map<std::string, std::string> getFileEncodings() const;
/// \name Private functions inherited from Inset class
//@{

View File

@ -32,8 +32,8 @@ extern char const * const lyx_version_info;
// Do not remove the comment below, so we get merge conflict in
// independent branches. Instead add your own.
#define LYX_FORMAT_LYX 569 // spitz: tablestyle buffer param
#define LYX_FORMAT_TEX2LYX 569
#define LYX_FORMAT_LYX 570 // spitz: biblatex bibencodings
#define LYX_FORMAT_TEX2LYX 570
#if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX
#ifndef _MSC_VER