anne/lyx_mirror
1
0
Fork 0
mirror of https://git.lyx.org/repos/lyx.git synced 2024-09-19 14:30:32 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add support for bib files encoding

Browse Source 
File format change.

Fixes: #6223
This commit is contained in:
Juergen Spitzmueller 2018-07-07 15:25:35 +02:00
parent 92c8a86407
commit e6e3777363
12 changed files with 265 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
development/FORMAT
View File

@ -7,6 +7,13 @@ changes happened in particular if possible. A good example would be
-----------------------
2018-07-07 Jürgen Spitzmüller <spitz@lyx.org>
* format incremented to 556: Support for bib file encoding.
- New param "encoding" for CommandInset bibtex
This will be used for biblatex output as bibencoding package option,
for BibTeX, \bibliography will be wrapped into
\bgroup\inputencoding{<enc>}...\egroup
2018-07-01 Jürgen Spitzmüller <spitz@lyx.org>
* format incremented to 555: Support varwidth's V tabular column type.
This effectively enables linebreaks, multipars and layout changes in

10
lib/doc/UserGuide.lyx
View File

@ -30183,6 +30183,16 @@ Content
\family default
drop box you can select whether to include all the entries in the database
in the document or just the cited references.
\change_inserted -712698321 1530969762
The
\family sans
Encoding
\family default
drop box lets you specify the encoding of the database(s), should this
differ from the encoding of the document.
\change_unchanged
\end_layout
\begin_layout Standard

102
lib/lyx2lyx/lyx_2_4.py
View File

@ -439,6 +439,104 @@ def revert_vcolumns(document):
add_to_preamble(document, ["\\usepackage{varwidth}"])
def revert_bibencoding(document):
" Revert bibliography encoding "
# Get cite engine
engine = "basic"
i = find_token(document.header, "\\cite_engine", 0)
if i == -1:
document.warning("Malformed document! Missing \\cite_engine")
else:
engine = get_value(document.header, "\\cite_engine", i)
# Check if biblatex
biblatex = False
if engine in ["biblatex", "biblatex-natbib"]:
biblatex = True
# Map lyx to latex encoding names
encodings = {
"utf8" : "utf8",
"utf8x" : "utf8x",
"armscii8" : "armscii8",
"iso8859-1" : "latin1",
"iso8859-2" : "latin2",
"iso8859-3" : "latin3",
"iso8859-4" : "latin4",
"iso8859-5" : "iso88595",
"iso8859-6" : "8859-6",
"iso8859-7" : "iso-8859-7",
"iso8859-8" : "8859-8",
"iso8859-9" : "latin5",
"iso8859-13" : "latin7",
"iso8859-15" : "latin9",
"iso8859-16" : "latin10",
"applemac" : "applemac",
"cp437" : "cp437",
"cp437de" : "cp437de",
"cp850" : "cp850",
"cp852" : "cp852",
"cp855" : "cp855",
"cp858" : "cp858",
"cp862" : "cp862",
"cp865" : "cp865",
"cp866" : "cp866",
"cp1250" : "cp1250",
"cp1251" : "cp1251",
"cp1252" : "cp1252",
"cp1255" : "cp1255",
"cp1256" : "cp1256",
"cp1257" : "cp1257",
"koi8-r" : "koi8-r",
"koi8-u" : "koi8-u",
"pt154" : "pt154",
"utf8-platex" : "utf8",
"ascii" : "ascii"
}
i = 0
bibresources = []
while (True):
i = find_token(document.body, "\\begin_inset CommandInset bibtex", i)
if i == -1:
break
j = find_end_of_inset(document.body, i)
if j == -1:
document.warning("Can't find end of bibtex inset at line %d!!" %(i))
i += 1
continue
encoding = get_quoted_value(document.body, "encoding", i, j)
if not encoding:
i += 1
continue
# remove encoding line
k = find_token(document.body, "encoding", i, j)
if k != -1:
del document.body[k]
# Re-find inset end line
j = find_end_of_inset(document.body, i)
if biblatex:
biblio_options = ""
h = find_token(document.header, "\\biblio_options", 0)
if h != -1:
biblio_options = get_value(document.header, "\\biblio_options", h)
if not "bibencoding" in biblio_options:
document.header[h] += ",bibencoding=%s" % encodings[encoding]
else:
bs = find_token(document.header, "\\biblatex_bibstyle", 0)
if bs == -1:
# this should not happen
document.warning("Malformed LyX document! No \\biblatex_bibstyle header found!")
else:
document.header[bs-1 : bs-1] = ["\\biblio_options bibencoding=" + encodings[encoding]]
else:
document.body[j+1:j+1] = put_cmd_in_ert("\\egroup")
document.body[i:i] = put_cmd_in_ert("\\bgroup\\inputencoding{" + encodings[encoding] + "}")
i = j + 1
##
# Conversion hub
#
@ -455,10 +553,12 @@ convert = [
[552, []],
[553, []],
[554, []],
[555, []]
[555, []],
[556, []]
]
revert = [
[555, [revert_bibencoding]],
[554, [revert_vcolumns]],
[553, [revert_stretchcolumn]],
[552, [revert_tuftecite]],

9
src/Buffer.cpp
View File

@ -2420,7 +2420,8 @@ BiblioInfo const & Buffer::bibInfo() const
}
void Buffer::registerBibfiles(FileNamePairList const & bf) const {
void Buffer::registerBibfiles(FileNamePairList const & bf) const
{
// We register the bib files in the master buffer,
// if there is one, but also in every single buffer,
// in case a child is compiled alone.
@ -2468,17 +2469,17 @@ void Buffer::checkIfBibInfoCacheIsValid() const
}
void Buffer::reloadBibInfoCache() const
void Buffer::reloadBibInfoCache(bool const force) const
{
// use the master's cache
Buffer const * const tmp = masterBuffer();
if (tmp != this) {
tmp->reloadBibInfoCache();
tmp->reloadBibInfoCache(force);
return;
}
checkIfBibInfoCacheIsValid();
if (d->bibinfo_cache_valid_)
if (d->bibinfo_cache_valid_ && !force)
return;
d->bibinfo_.clear();

2
src/Buffer.h
View File

@ -505,7 +505,7 @@ public:
/// whether the cache is valid. If so, we do nothing. If not, then we
/// reload all the BibTeX info.
/// Note that this operates on the master document.
void reloadBibInfoCache() const;
void reloadBibInfoCache(bool const force = false) const;
/// \return the bibliography information for this buffer's master,
/// or just for it, if it isn't a child.
BiblioInfo const & masterBibInfo() const;

5
src/BufferParams.cpp
View File

@ -2352,6 +2352,11 @@ bool BufferParams::writeLaTeX(otexstream & os, LaTeXFeatures & features,
opts += delim + "backend=bibtex";
delim = ",";
}
if (!bib_encoding.empty() && encodings.fromLyXName(bib_encoding)) {
opts += delim + "bibencoding="
+ encodings.fromLyXName(bib_encoding)->latexName();
delim = ",";
}
if (!biblio_opts.empty())
opts += delim + biblio_opts;
if (!opts.empty())

9
src/BufferParams.h
View File

@ -499,6 +499,10 @@ public:
std::string biblatex_bibstyle;
/// The biblatex citation style
std::string biblatex_citestyle;
/// Set the bib file encoding (for biblatex)
void setBibEncoding(std::string const & s) { bib_encoding = s; }
/// Get the bib file encoding (for biblatex)
std::string const & bibEncoding() const { return bib_encoding; }
/// options for pdf output
PDFOptions & pdfoptions();
@ -572,6 +576,11 @@ private:
CiteEngineType cite_engine_type_;
/// the default BibTeX style file for the document
std::string biblio_style;
/// The encoding of the bib files, for Biblatex
/// (only one supported currently)
// FIXME: biblatex 3.12 introduces per-file
// encoding options. Update once that's spread.
std::string bib_encoding;
/// Split bibliography?
bool use_bibtopic;
///

25
src/frontends/qt4/GuiBibtex.cpp
View File

@ -18,6 +18,7 @@
#include "Buffer.h"
#include "BufferParams.h"
#include "CiteEnginesList.h"
#include "Encoding.h"
#include "FuncRequest.h"
#include "LyXRC.h"
#include "qt_helpers.h"
@ -85,6 +86,8 @@ GuiBibtex::GuiBibtex(GuiView & lv)
this, SLOT(rescanClicked()));
connect(biblatexOptsLE, SIGNAL(textChanged(QString)),
this, SLOT(change_adaptor()));
connect(bibEncodingCO, SIGNAL(activated(int)),
this, SLOT(change_adaptor()));
add_ = new GuiBibtexAddDialog(this);
add_bc_.setPolicy(ButtonPolicy::OkCancelPolicy);
@ -123,9 +126,23 @@ GuiBibtex::GuiBibtex(GuiView & lv)
bc().addReadOnly(styleCB);
bc().addReadOnly(bibtocCB);
bc().addReadOnly(addBibPB);
bc().addReadOnly(bibEncodingCO);
// Delete/Up/Down are handled with more conditions in
// databaseChanged().
// Always put the default encoding in the first position.
bibEncodingCO->addItem(qt_("Document Encoding"), "default");
QMap<QString, QString> encodinglist;
for (auto const & encvar : encodings) {
if (!encvar.unsafe() && !encvar.guiName().empty())
encodinglist.insert(qt_(encvar.guiName()), toqstr(encvar.name()));
}
QMap<QString, QString>::const_iterator it = encodinglist.constBegin();
while (it != encodinglist.constEnd()) {
bibEncodingCO->addItem(it.key(), it.value());
++it;
}
// Make sure the delete/up/down buttons are disabled if necessary.
databaseChanged();
}
@ -350,6 +367,12 @@ void GuiBibtex::updateContents()
btprint = from_ascii("btPrintCited");
btPrintCO->setCurrentIndex(btPrintCO->findData(toqstr(btprint)));
docstring encoding = params_["encoding"];
if (encoding.empty())
// default
encoding = from_ascii("default");
bibEncodingCO->setCurrentIndex(bibEncodingCO->findData(toqstr(encoding)));
// Only useful for biblatex
biblatexOptsLA->setVisible(biblatex);
biblatexOptsLE->setVisible(biblatex);
@ -421,6 +444,8 @@ void GuiBibtex::applyView()
params_["biblatexopts"] = qstring_to_ucs4(biblatexOptsLE->text());
params_["btprint"] = qstring_to_ucs4(btPrintCO->itemData(btPrintCO->currentIndex()).toString());
params_["encoding"] = qstring_to_ucs4(bibEncodingCO->itemData(bibEncodingCO->currentIndex()).toString());
}

82
src/frontends/qt4/ui/BibtexUi.ui
View File

@ -6,8 +6,8 @@
<rect>
<x>0</x>
<y>0</y>
<width>361</width>
<height>397</height>
<width>368</width>
<height>441</height>
</rect>
</property>
<property name="windowTitle">
@ -17,7 +17,7 @@
<bool>true</bool>
</property>
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="0" colspan="2">
<item row="0" column="0">
<widget class="QLabel" name="databaseLA">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
@ -39,7 +39,7 @@
</property>
</widget>
</item>
<item row="1" column="0" rowspan="6" colspan="2">
<item row="1" column="0" rowspan="5" colspan="2">
<widget class="QListWidget" name="databaseLW"/>
</item>
<item row="1" column="2">
@ -82,7 +82,7 @@
</property>
</widget>
</item>
<item row="5" column="2">
<item row="5" column="2" rowspan="2">
<spacer>
<property name="orientation">
<enum>Qt::Vertical</enum>
@ -98,7 +98,7 @@
</property>
</spacer>
</item>
<item row="6" column="2" rowspan="2">
<item row="7" column="2" rowspan="2">
<widget class="QPushButton" name="rescanPB">
<property name="toolTip">
<string>Scan for new databases and styles</string>
@ -108,7 +108,7 @@
</property>
</widget>
</item>
<item row="7" column="0">
<item row="8" column="0">
<widget class="QLabel" name="styleLA">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
@ -127,7 +127,7 @@
</property>
</widget>
</item>
<item row="8" column="0" colspan="2">
<item row="9" column="0" colspan="2">
<widget class="QComboBox" name="styleCB">
<property name="toolTip">
<string>Choose a style file</string>
@ -143,7 +143,7 @@
</property>
</widget>
</item>
<item row="8" column="2">
<item row="9" column="2">
<widget class="QPushButton" name="stylePB">
<property name="toolTip">
<string>Choose a style file</string>
@ -156,7 +156,7 @@
</property>
</widget>
</item>
<item row="9" column="0">
<item row="10" column="0">
<widget class="QLabel" name="btPrintLA">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
@ -175,7 +175,7 @@
</property>
</widget>
</item>
<item row="9" column="1">
<item row="10" column="1">
<widget class="QComboBox" name="btPrintCO">
<property name="toolTip">
<string>This bibliography section contains...</string>
@ -197,7 +197,7 @@
</item>
</widget>
</item>
<item row="9" column="2" rowspan="2">
<item row="10" column="2">
<spacer>
<property name="orientation">
<enum>Qt::Vertical</enum>
@ -213,17 +213,7 @@
</property>
</spacer>
</item>
<item row="10" column="0" colspan="2">
<widget class="QCheckBox" name="bibtocCB">
<property name="toolTip">
<string>Add bibliography to the table of contents</string>
</property>
<property name="text">
<string>Add bibliography to &amp;TOC</string>
</property>
</widget>
</item>
<item row="11" column="0" colspan="3">
<item row="12" column="0" colspan="3">
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QLabel" name="biblatexOptsLA">
@ -244,7 +234,7 @@
</item>
</layout>
</item>
<item row="12" column="0" colspan="3">
<item row="13" column="0" colspan="3">
<layout class="QHBoxLayout">
<property name="spacing">
<number>6</number>
@ -302,6 +292,50 @@
</item>
</layout>
</item>
<item row="6" column="0" rowspan="2" colspan="2">
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QLabel" name="bibEncodingLA">
<property name="text">
<string>&amp;Encoding:</string>
</property>
<property name="buddy">
<cstring>bibEncodingCO</cstring>
</property>
</widget>
</item>
<item>
<widget class="QComboBox" name="bibEncodingCO">
<property name="toolTip">
<string>If your bibliography databases use a different encoding than the LyX document, specify it here</string>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
<item row="11" column="0" colspan="2">
<widget class="QCheckBox" name="bibtocCB">
<property name="toolTip">
<string>Add bibliography to the table of contents</string>
</property>
<property name="text">
<string>Add bibliography to &amp;TOC</string>
</property>
</widget>
</item>
</layout>
</widget>
<tabstops>

44
src/insets/InsetBibtex.cpp
View File

@ -26,6 +26,7 @@
#include "FuncRequest.h"
#include "FuncStatus.h"
#include "LaTeXFeatures.h"
#include "output_latex.h"
#include "output_xhtml.h"
#include "OutputParams.h"
#include "PDFOptions.h"
@ -69,6 +70,7 @@ ParamInfo const & InsetBibtex::findInfo(string const & /* cmdName */)
param_info_.add("btprint", ParamInfo::LATEX_OPTIONAL);
param_info_.add("bibfiles", ParamInfo::LATEX_REQUIRED);
param_info_.add("options", ParamInfo::LYX_INTERNAL);
param_info_.add("encoding", ParamInfo::LYX_INTERNAL);
param_info_.add("biblatexopts", ParamInfo::LATEX_OPTIONAL);
}
return param_info_;
@ -326,6 +328,19 @@ void InsetBibtex::latex(otexstream & os, OutputParams const & runparams) const
_("There are spaces in the path to your BibTeX style file.\n"
"BibTeX will be unable to find it."));
}
// Encoding
bool encoding_switched = false;
Encoding const * const save_enc = runparams.encoding;
docstring const encoding = getParam("encoding");
if (!encoding.empty() && encoding != from_ascii("default")) {
Encoding const * const enc = encodings.fromLyXName(to_ascii(encoding));
if (enc != runparams.encoding) {
os << "\\bgroup";
switchEncoding(os.os(), buffer().params(), runparams, *enc, true);
runparams.encoding = enc;
encoding_switched = true;
}
}
// Handle the bibtopic case
if (!db_out.empty() && buffer().masterParams().useBibtopic()) {
os << "\\begin{btSect}";
@ -357,6 +372,13 @@ void InsetBibtex::latex(otexstream & os, OutputParams const & runparams) const
}
os << "\\bibliography{" << getStringFromVector(db_out) << "}\n";
}
if (encoding_switched){
// Switch back
switchEncoding(os.os(), buffer().params(),
runparams, *save_enc, true, true);
os << "\\egroup" << breakln;
runparams.encoding = save_enc;
}
}
}
@ -645,9 +667,9 @@ void InsetBibtex::parseBibTeXFiles(FileNameList & checkedFiles) const
// bibtex does.
//
// Officially bibtex does only support ASCII, but in practice
// you can use the encoding of the main document as long as
// some elements like keys and names are pure ASCII. Therefore
// we convert the file from the buffer encoding.
// you can use any encoding as long as some elements like keys
// and names are pure ASCII. We support specifying an encoding,
// and we convert the file from that (default is buffer encoding).
// We don't restrict keys to ASCII in LyX, since our own
// InsetBibitem can generate non-ASCII keys, and nonstandard
// 8bit clean bibtex forks exist.
@ -665,8 +687,12 @@ void InsetBibtex::parseBibTeXFiles(FileNameList & checkedFiles) const
else
// record that we check this.
checkedFiles.push_back(bibfile);
string encoding = buffer().masterParams().encoding().iconvName();
string const ienc = to_ascii(params()["encoding"]);
if (!ienc.empty() && ienc != "default" && encodings.fromLyXName(ienc))
encoding = encodings.fromLyXName(ienc)->iconvName();
ifdocstream ifs(bibfile.toFilesystemEncoding().c_str(),
ios_base::in, buffer().masterParams().encoding().iconvName());
ios_base::in, encoding);
char_type ch;
VarMap strings;
@ -892,8 +918,16 @@ void InsetBibtex::validate(LaTeXFeatures & features) const
}
void InsetBibtex::updateBuffer(ParIterator const &, UpdateType) {
void InsetBibtex::updateBuffer(ParIterator const &, UpdateType)
{
buffer().registerBibfiles(getBibFiles());
// record encoding of bib files for biblatex
string const enc = (params()["encoding"] == from_ascii("default")) ?
string() : to_ascii(params()["encoding"]);
if (buffer().params().bibEncoding() != enc) {
buffer().params().setBibEncoding(enc);
buffer().reloadBibInfoCache(true);
}
}

3
src/tex2lyx/TODO.txt
View File

@ -60,6 +60,9 @@ Format LaTeX feature LyX feature
\begin{landscape}...\end{landscape} \begin_inset Flex Landscape
with longtable content: <features rotate ="90"...>
555 V column type (varwidth package) Automatically detected with newlines, paragraph breaks and environment content in cells of rows
556 Bib files encoding
\begin_inset CommandInset bibtex Biblatex: bibencoding=<encoding> [latex name]
encoding=<encoding> [lyx name] BibTeX: \bgroup\inputencoding{<encoding>}...\egroup

4
src/version.h
View File

@ -32,8 +32,8 @@ extern char const * const lyx_version_info;
// Do not remove the comment below, so we get merge conflict in
// independent branches. Instead add your own.
#define LYX_FORMAT_LYX 555 // spitz: varwidth V columns
#define LYX_FORMAT_TEX2LYX 555
#define LYX_FORMAT_LYX 556 // spitz: bib encoding
#define LYX_FORMAT_TEX2LYX 556
#if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX
#ifndef _MSC_VER