georg baum:

- add encoding arg to idocfstream
- open .bib files with idocfstream
- add ascii_lowercase that takes docstring

git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@16281 a592a061-630c-0410-9148-cb99ea01b6c8

src/insets/insetbibtex.C

@@ -16,6 +16,7 @@
 #include "bufferparams.h"
 #include "dispatchresult.h"
 #include "debug.h"
+#include "encoding.h"
 #include "funcrequest.h"
 #include "gettext.h"
 #include "LaTeXFeatures.h"
@@ -32,9 +33,6 @@
 
 #include <boost/tokenizer.hpp>
 
-#include <fstream>
-#include <sstream>
-
 
 namespace lyx {
 
@@ -66,7 +64,6 @@ namespace os = support::os;
 using std::endl;
 using std::getline;
 using std::string;
-using std::ifstream;
 using std::ostream;
 using std::pair;
 using std::vector;
@@ -342,26 +339,36 @@ void InsetBibtex::fillWithBibKeys(Buffer const & buffer,
 		// files. All it does is to look for lines starting
 		// in @ and not being @preamble and @string entries.
 		// It does NOT do any syntax checking!
-		ifstream ifs(it->toFilesystemEncoding().c_str());
-		string linebuf0;
+
+		// Officially bibtex does only support ASCII, but in practice
+		// you can use the encoding of the main document as long as
+		// some elements like keys and names are pure ASCII. Therefore
+		// we convert the file from the buffer encoding.
+		idocfstream ifs(it->toFilesystemEncoding().c_str(),
+		                std::ios_base::in,
+		                buffer.params().encoding().iconvName());
+		docstring linebuf0;
 		while (getline(ifs, linebuf0)) {
-			string linebuf = trim(linebuf0);
+			docstring linebuf = trim(linebuf0);
 			if (linebuf.empty()) continue;
-			if (prefixIs(linebuf, "@")) {
+			if (prefixIs(linebuf, from_ascii("@"))) {
 				linebuf = subst(linebuf, '{', '(');
-				string tmp;
+				docstring tmp;
 				linebuf = split(linebuf, tmp, '(');
 				tmp = ascii_lowercase(tmp);
-				if (!prefixIs(tmp, "@string")
-				    && !prefixIs(tmp, "@preamble")) {
+				if (!prefixIs(tmp, from_ascii("@string")) &&
+				    !prefixIs(tmp, from_ascii("@preamble"))) {
 					linebuf = split(linebuf, tmp, ',');
 					tmp = ltrim(tmp, " \t");
 					if (!tmp.empty()) {
-						keys.push_back(pair<string,string>(tmp,string()));
+						// to_ascii because bibtex keys may
+						// only consist of ASCII characters
+						keys.push_back(pair<string, string>(to_ascii(tmp), string()));
 					}
 				}
 			} else if (!keys.empty()) {
-				keys.back().second += linebuf + "\n";
+				// FIXME UNICODE
+				keys.back().second += to_utf8(linebuf + '\n');
 			}
 		}
 	}

src/support/docstream.C

@@ -26,8 +26,6 @@ using std::string;
 
 namespace {
 
-char const * utf8_codeset = "UTF-8";
-
 // We use C IO throughout this file, because the facets might be used with
 // lyxerr in the future.
 
@@ -199,20 +197,21 @@ const char * iconv_codecvt_facet_exception::what() const throw()
 }
 
 
-idocfstream::idocfstream() : base()
+idocfstream::idocfstream(string const & encoding) : base()
 {
 	std::locale global;
-	std::locale locale(global, new iconv_codecvt_facet(utf8_codeset, in));
+	std::locale locale(global, new iconv_codecvt_facet(encoding, in));
 	imbue(locale);
 }
 
 	
-idocfstream::idocfstream(const char* s, std::ios_base::openmode mode)
+idocfstream::idocfstream(const char* s, std::ios_base::openmode mode,
+                         string const & encoding)
 	: base()
 {
 	// We must imbue the stream before openening the file
 	std::locale global;
-	std::locale locale(global, new iconv_codecvt_facet(utf8_codeset, in));
+	std::locale locale(global, new iconv_codecvt_facet(encoding, in));
 	imbue(locale);
 	open(s, mode);
 }

src/support/docstream.h

@@ -45,9 +45,10 @@ typedef std::basic_ostream<char_type> odocstream;
 class idocfstream : public std::basic_ifstream<char_type> {
 	typedef std::basic_ifstream<char_type> base;
 public:
-	idocfstream();
+	idocfstream(std::string const & encoding = "UTF-8");
 	explicit idocfstream(const char* s,
-		std::ios_base::openmode mode = std::ios_base::in);
+		std::ios_base::openmode mode = std::ios_base::in,
+		std::string const & encoding = "UTF-8");
 	~idocfstream() {}
 };
 

src/support/lstrings.C

@@ -373,8 +373,8 @@ struct local_uppercase {
 	}
 };
 
-struct local_ascii_lowercase {
-	char operator()(char c) const {
+template<typename Char> struct local_ascii_lowercase {
+	Char operator()(Char c) const {
 		return ascii_tolower(c);
 	}
 };
@@ -409,7 +409,16 @@ string const ascii_lowercase(string const & a)
 {
 	string tmp(a);
 	transform(tmp.begin(), tmp.end(), tmp.begin(),
-		  local_ascii_lowercase());
+		  local_ascii_lowercase<char>());
+	return tmp;
+}
+
+
+docstring const ascii_lowercase(docstring const & a)
+{
+	docstring tmp(a);
+	transform(tmp.begin(), tmp.end(), tmp.begin(),
+		  local_ascii_lowercase<char_type>());
 	return tmp;
 }
 

src/support/lstrings.h

@@ -89,6 +89,7 @@ char_type uppercase(char_type c);
 
 /// same as lowercase(), but ignores locale
 std::string const ascii_lowercase(std::string const &);
+docstring const ascii_lowercase(docstring const &);
 
 ///
 std::string const lowercase(std::string const &);