Fix clipboard/selection encoding

* src/frontends/qt[34]/qt_helpers.[Ch]
	(toqstr): add variant for docstring
	(qstring_to_ucs4): Use docstring and port from qt4 to qt3

	* Many other files: Many std::string -> lyx::docstring conversions

	* src/support/lstrings.[Ch]
	(subst): Add variant for docstring and char_type
	(externalLineEnding): std::string -> lyx::docstring
	(internalLineEnding): std::string -> lyx::docstring


git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@14871 a592a061-630c-0410-9148-cb99ea01b6c8

src/BufferView_pimpl.C

@@ -83,6 +83,7 @@
 using lyx::frontend::Clipboard;
 using lyx::frontend::Gui;
 
+using lyx::docstring;
 using lyx::pos_type;
 
 using lyx::support::addPath;
@@ -460,7 +461,7 @@ void BufferView::Pimpl::scroll(int /*lines*/)
 
 void BufferView::Pimpl::selectionRequested()
 {
-	static string sel;
+	static docstring sel;
 
 	if (!available())
 		return;

src/buffer.C

@@ -93,6 +93,7 @@ namespace io = boost::iostreams;
 #include <fstream>
 
 
+using lyx::docstring;
 using lyx::pos_type;
 using lyx::pit_type;
 
@@ -504,13 +505,13 @@ bool Buffer::readDocument(LyXLex & lex)
 // needed to insert the selection
 void Buffer::insertStringAsLines(ParagraphList & pars,
 	pit_type & pit, pos_type & pos,
-	LyXFont const & fn, string const & str, bool autobreakrows)
+	LyXFont const & fn, docstring const & str, bool autobreakrows)
 {
 	LyXFont font = fn;
 
 	// insert the string, don't insert doublespace
 	bool space_inserted = true;
-	for (string::const_iterator cit = str.begin();
+	for (docstring::const_iterator cit = str.begin();
 	    cit != str.end(); ++cit) {
 		Paragraph & par = pars[pit];
 		if (*cit == '\n') {
@@ -541,9 +542,11 @@ void Buffer::insertStringAsLines(ParagraphList & pars,
 				}
 				space_inserted = true;
 			}
+/* FIXME: not needed anymore?
 		} else if (!isPrintable(*cit)) {
 			// Ignore unprintables
 			continue;
+*/
 		} else {
 			// just insert the character
 			par.insertChar(pos, *cit, font);

src/buffer.h

@@ -107,7 +107,7 @@ public:
 	///
 	void insertStringAsLines(ParagraphList & plist,
 		lyx::pit_type &, lyx::pos_type &,
-		LyXFont const &, std::string const &, bool);
+		LyXFont const &, lyx::docstring const &, bool);
 	///
 	ParIterator getParFromID(int id) const;
 	/// do we have a paragraph with this id?

src/cursor.C

@@ -54,6 +54,7 @@
 #include <limits>
 
 using lyx::char_type;
+using lyx::docstring;
 using lyx::pit_type;
 
 using std::string;
@@ -1114,10 +1115,10 @@ void LCursor::errorMessage(string const & msg) const
 }
 
 
-string LCursor::selectionAsString(bool label) const
+docstring LCursor::selectionAsString(bool label) const
 {
 	if (!selection())
-		return string();
+		return docstring();
 
 	if (inTexted()) {
 		Buffer const & buffer = *bv().buffer();
@@ -1130,7 +1131,7 @@ string LCursor::selectionAsString(bool label) const
 		size_t const endpos = selEnd().pos();
 
 		if (startpit == endpit)
-			return pars[startpit].asString(buffer, startpos, endpos, label);
+			return lyx::from_utf8(pars[startpit].asString(buffer, startpos, endpos, label));
 
 		// First paragraph in selection
 		string result = pars[startpit].
@@ -1145,13 +1146,13 @@ string LCursor::selectionAsString(bool label) const
 		// Last paragraph in selection
 		result += pars[endpit].asString(buffer, 0, endpos, label);
 
-		return result;
+		return lyx::from_utf8(result);
 	}
 
 	if (inMathed())
-		return lyx::cap::grabSelection(*this);
+		return lyx::from_utf8(lyx::cap::grabSelection(*this));
 
-	return string();
+	return docstring();
 }
 
 

src/cursor.h

@@ -86,7 +86,7 @@ public:
 	///
 	void selHandle(bool selecting);
 	//
-	std::string selectionAsString(bool label) const;
+	lyx::docstring selectionAsString(bool label) const;
 	///
 	std::string currentState();
 

src/frontends/Clipboard.h

@@ -14,7 +14,7 @@
 #ifndef BASE_CLIPBOARD_H
 #define BASE_CLIPBOARD_H
 
-#include <string>
+#include "support/docstring.h"
 
 namespace lyx {
 namespace frontend {
@@ -32,13 +32,13 @@ public:
 	 * This should be called when the user requests to paste from the
 	 * clipboard.
 	 */
-	virtual std::string const get() const = 0;
+	virtual docstring const get() const = 0;
 	/**
 	 * Fill the window system clipboard.
 	 * This should be called when the user requests to cut or copy to
 	 * the clipboard.
 	 */
-	virtual void put(std::string const &) = 0;
+	virtual void put(docstring const &) = 0;
 };
 
 } // namespace frontend

src/frontends/Selection.h

@@ -14,7 +14,7 @@
 #ifndef BASE_SELECTION_H
 #define BASE_SELECTION_H
 
-#include <string>
+#include "support/docstring.h"
 
 namespace lyx {
 namespace frontend {
@@ -36,13 +36,13 @@ public:
 	 * This should be called when the user presses the middle mouse
 	 * button.
 	 */
-	virtual std::string const get() const = 0;
+	virtual docstring const get() const = 0;
 	/**
 	 * Fill the X selection.
 	 * Does nothing on systems that don't have a selection.
 	 * This should be called whenever some text is highlighted.
 	 */
-	virtual void put(std::string const &) = 0;
+	virtual void put(docstring const &) = 0;
 };
 
 } // namespace frontend

src/frontends/gtk/GuiClipboard.C

@@ -31,29 +31,23 @@ using std::string;
 namespace lyx {
 namespace frontend {
 
-// ENCODING: Gtk::Clipboard returns UTF-8, we assume that the backend
+docstring const GuiClipboard::get() const
-// wants ISO-8859-1 and convert it to that.
-// FIXME: Wrong!
-string const GuiClipboard::get() const
 {
 	Glib::RefPtr<Gtk::Clipboard> clipboard =
 		Gtk::Clipboard::get(GDK_SELECTION_CLIPBOARD);
-	string const str = Glib::convert_with_fallback(
+	string const str = clipboard->wait_for_text();
-			clipboard->wait_for_text(), "ISO-8859-1", "UTF-8");
 	lyxerr[Debug::ACTION] << "GuiClipboard::get: " << str << endl;
-	return str;
+	return lyx::from_utf8(str);
 }
 
 
-// ENCODING: we assume that the backend passes us ISO-8859-1 and
+void GuiClipboard::put(docstring const & str)
-// convert from that to UTF-8 before passing to GTK
-// FIXME: Wrong!
-void GuiClipboard::put(string const & str)
 {
-	lyxerr[Debug::ACTION] << "GuiClipboard::put: " << str << endl;
+	string const utf8 = lyx::to_utf8(str);
+	lyxerr[Debug::ACTION] << "GuiClipboard::put: " << utf8 << endl;
 	Glib::RefPtr<Gtk::Clipboard> clipboard =
 		Gtk::Clipboard::get(GDK_SELECTION_CLIPBOARD);
-	clipboard->set_text(Glib::convert(str, "UTF-8", "ISO-8859-1"));
+	clipboard->set_text(utf8);
 }
 
 } // namespace frontend

src/frontends/gtk/GuiClipboard.h

@@ -29,9 +29,9 @@ public:
 	 */
 	//@{
 
-	std::string const get() const;
+	docstring const get() const;
 
-	void put(std::string const & str);
+	void put(docstring const & str);
 	//@}
 };
 

src/frontends/gtk/GuiSelection.C

@@ -31,29 +31,23 @@ using std::string;
 namespace lyx {
 namespace frontend {
 
-// ENCODING: Gtk::Clipboard returns UTF-8, we assume that the backend
+docstring const GuiSelection::get() const
-// wants ISO-8859-1 and convert it to that.
-// FIXME: Wrong!
-string const GuiSelection::get() const
 {
 	Glib::RefPtr<Gtk::Clipboard> clipboard =
 		Gtk::Clipboard::get(GDK_SELECTION_PRIMARY);
-	string const str = Glib::convert_with_fallback(
+	string const str = clipboard->wait_for_text();
-			clipboard->wait_for_text(), "ISO-8859-1", "UTF-8");
 	lyxerr[Debug::ACTION] << "GuiClipboard::get: " << str << endl;
-	return str;
+	return lyx::from_utf8(str);
 }
 
 
-// ENCODING: we assume that the backend passes us ISO-8859-1 and
+void GuiSelection::put(docstring const & str)
-// convert from that to UTF-8 before passing to GTK
-// FIXME: Wrong!
-void GuiSelection::put(string const & str)
 {
-	lyxerr[Debug::ACTION] << "GuiClipboard::put: " << str << endl;
+	string const utf8 = lyx::to_utf8(str);
+	lyxerr[Debug::ACTION] << "GuiClipboard::put: " << utf8 << endl;
 	Glib::RefPtr<Gtk::Clipboard> clipboard =
 		Gtk::Clipboard::get(GDK_SELECTION_PRIMARY);
-	clipboard->set_text(Glib::convert(str, "UTF-8", "ISO-8859-1"));
+	clipboard->set_text(utf8);
 }
 
 } // namespace frontend

src/frontends/gtk/GuiSelection.h

@@ -40,9 +40,9 @@ public:
 		old_work_area_->haveSelection(own);
 	}
 
-	std::string const get() const;
+	docstring const get() const;
 
-	void put(std::string const & str);
+	void put(docstring const & str);
 	//@}
 
 private:

src/frontends/qt3/GuiClipboard.C

@@ -26,26 +26,25 @@ using lyx::support::internalLineEnding;
 using lyx::support::externalLineEnding;
 
 using std::endl;
-using std::string;
 
 namespace lyx {
 namespace frontend {
 
-string const GuiClipboard::get() const
+docstring const GuiClipboard::get() const
 {
 	QString const str = qApp->clipboard()->text(QClipboard::Clipboard);
-	lyxerr[Debug::ACTION] << "GuiClipboard::get: " << (const char*) str
+	lyxerr[Debug::ACTION] << "GuiClipboard::get: " << fromqstr(str)
 	                      << endl;
 	if (str.isNull())
-		return string();
+		return docstring();
 
-	return internalLineEnding(fromqstr(str));
+	return internalLineEnding(qstring_to_ucs4(str));
 }
 
 
-void GuiClipboard::put(string const & str)
+void GuiClipboard::put(docstring const & str)
 {
-	lyxerr[Debug::ACTION] << "GuiClipboard::put: " << str << endl;
+	lyxerr[Debug::ACTION] << "GuiClipboard::put: " << lyx::to_utf8(str) << endl;
 
 	qApp->clipboard()->setText(toqstr(externalLineEnding(str)),
 	                           QClipboard::Clipboard);

src/frontends/qt3/GuiClipboard.h

@@ -29,9 +29,9 @@ public:
 	 */
 	//@{
 
-	std::string const get() const;
+	docstring const get() const;
 
-	void put(std::string const & str);
+	void put(docstring const & str);
 	//@}
 };
 

src/frontends/qt3/GuiSelection.C

@@ -26,26 +26,25 @@ using lyx::support::internalLineEnding;
 using lyx::support::externalLineEnding;
 
 using std::endl;
-using std::string;
 
 namespace lyx {
 namespace frontend {
 
-string const GuiSelection::get() const
+docstring const GuiSelection::get() const
 {
 	QString const str = qApp->clipboard()->text(QClipboard::Selection);
 	lyxerr[Debug::ACTION] << "GuiSelection::get: " << (const char*) str
 	                      << endl;
 	if (str.isNull())
-		return string();
+		return docstring();
 
-	return internalLineEnding(fromqstr(str));
+	return internalLineEnding(qstring_to_ucs4(str));
 }
 
 
-void GuiSelection::put(string const & str)
+void GuiSelection::put(docstring const & str)
 {
-	lyxerr[Debug::ACTION] << "GuiSelection::put: " << str << endl;
+	lyxerr[Debug::ACTION] << "GuiSelection::put: " << lyx::to_utf8(str) << endl;
 
 	qApp->clipboard()->setText(toqstr(externalLineEnding(str)),
 	                           QClipboard::Selection);

src/frontends/qt3/GuiSelection.h

@@ -40,9 +40,9 @@ public:
 		old_work_area_->haveSelection(own);
 	}
 
-	std::string const get() const;
+	docstring const get() const;
 
-	void put(std::string const & str);
+	void put(docstring const & str);
 	//@}
 
 private:

src/frontends/qt3/qt_helpers.C

@@ -19,6 +19,7 @@
 
 #include "support/lstrings.h"
 #include "support/convert.h"
+#include "support/unicode.h"
 
 #include <qcombobox.h>
 #include <qlineedit.h>
@@ -28,6 +29,8 @@
 
 
 using lyx::support::isStrDbl;
+using lyx::char_type;
+using lyx::docstring;
 
 using std::make_pair;
 using std::string;
@@ -114,6 +117,15 @@ QString const toqstr(string const & str)
 }
 
 
+QString const toqstr(docstring const & str)
+{
+	std::vector<unsigned short> ucs2 =
+		ucs4_to_ucs2(str.c_str(), str.length());
+	ucs2.push_back('\0');
+	return QString::fromUcs2(&ucs2[0]);
+}
+
+
 QString const qt_(char const * str)
 {
 	return toqstr(_(str));
@@ -134,6 +146,15 @@ string const fromqstr(QString const & str)
 }
 
 
+docstring const qstring_to_ucs4(QString const & str)
+{
+	unsigned short const * const ucs2 = str.ucs2();
+	std::vector<char_type> const ucs4 = ucs2_to_ucs4(
+		std::vector<unsigned short>(ucs2, ucs2 + str.length()));
+	return docstring(ucs4.begin(), ucs4.end());
+}
+
+
 string const formatted(string const & text, int w)
 {
 	string sout;

src/frontends/qt3/qt_helpers.h

@@ -15,6 +15,7 @@
 #include <utility>
 
 #include "lyxlength.h"
+#include "support/docstring.h"
 
 class LengthCombo;
 class QComboBox;
@@ -55,6 +56,12 @@ QString const toqstr(char const * str);
 QString const toqstr(std::string const & str);
 
 
+/**
+ * toqstr - convert UCS4 encoded docstring to QString
+ */
+QString const toqstr(lyx::docstring const & str);
+
+
 /**
  * qt_ - i18nize string and convert to unicode
  *
@@ -78,4 +85,10 @@ QString const qt_(std::string const & str);
  */
 std::string const fromqstr(QString const & str);
 
+
+/**
+ * qstring_to_ucs4 - convert QString to UCS4 encoded docstring
+ */
+lyx::docstring const qstring_to_ucs4(QString const & str);
+
 #endif // QTHELPERS_H

src/frontends/qt4/GuiClipboard.C

@@ -26,26 +26,25 @@ using lyx::support::internalLineEnding;
 using lyx::support::externalLineEnding;
 
 using std::endl;
-using std::string;
 
 namespace lyx {
 namespace frontend {
 
-string const GuiClipboard::get() const
+docstring const GuiClipboard::get() const
 {
 	QString const str = qApp->clipboard()->text(QClipboard::Clipboard);
 	lyxerr[Debug::ACTION] << "GuiClipboard::get: " << fromqstr(str)
 	                      << endl;
 	if (str.isNull())
-		return string();
+		return docstring();
 
-	return internalLineEnding(fromqstr(str));
+	return internalLineEnding(qstring_to_ucs4(str));
 }
 
 
-void GuiClipboard::put(string const & str)
+void GuiClipboard::put(docstring const & str)
 {
-	lyxerr[Debug::ACTION] << "GuiClipboard::put: " << str << endl;
+	lyxerr[Debug::ACTION] << "GuiClipboard::put: " << lyx::to_utf8(str) << endl;
 
 	qApp->clipboard()->setText(toqstr(externalLineEnding(str)),
 	                           QClipboard::Clipboard);

src/frontends/qt4/GuiClipboard.h

@@ -30,8 +30,8 @@ public:
 	/** Clipboard overloaded methods
 	 */
 	//@{
-	std::string const get() const;
+	docstring const get() const;
-	void put(std::string const & str);
+	void put(docstring const & str);
 	//@}
 };
 

src/frontends/qt4/GuiSelection.C

@@ -26,7 +26,6 @@ using lyx::support::internalLineEnding;
 using lyx::support::externalLineEnding;
 
 using std::endl;
-using std::string;
 
 namespace lyx {
 namespace frontend {
@@ -43,21 +42,21 @@ void GuiSelection::haveSelection(bool own)
 }
 
 
-string const GuiSelection::get() const
+docstring const GuiSelection::get() const
 {
 	QString const str = qApp->clipboard()->text(QClipboard::Selection);
 	lyxerr[Debug::ACTION] << "GuiSelection::get: " << fromqstr(str)
 	                      << endl;
 	if (str.isNull())
-		return string();
+		return docstring();
 
-	return internalLineEnding(fromqstr(str));
+	return internalLineEnding(qstring_to_ucs4(str));
 }
 
 
-void GuiSelection::put(string const & str)
+void GuiSelection::put(docstring const & str)
 {
-	lyxerr[Debug::ACTION] << "GuiSelection::put: " << str << endl;
+	lyxerr[Debug::ACTION] << "GuiSelection::put: " << lyx::to_utf8(str) << endl;
 
 	qApp->clipboard()->setText(toqstr(externalLineEnding(str)),
 	                           QClipboard::Selection);

src/frontends/qt4/GuiSelection.h

@@ -31,8 +31,8 @@ public:
 	 */
 	//@{
 	void haveSelection(bool own);
-	std::string const get() const;
+	docstring const get() const;
-	void put(std::string const & str);
+	void put(docstring const & str);
 	//@}
 };
 

src/frontends/qt4/qt_helpers.C

@@ -31,6 +31,7 @@
 
 using lyx::support::isStrDbl;
 using lyx::char_type;
+using lyx::docstring;
 
 using std::vector;
 using std::make_pair;
@@ -130,7 +131,7 @@ QString const ucs4_to_qstring(char_type const * str, size_t ls)
 }
 
 
-QString const ucs4_to_qstring(vector<char_type> const & ucs4)
+QString const toqstr(docstring const & ucs4)
 {
 	QString s;
 	size_t const ls = ucs4.size();
@@ -142,12 +143,12 @@ QString const ucs4_to_qstring(vector<char_type> const & ucs4)
 }
 
 
-vector<char_type> qstring_to_ucs4(QString const & qstr)
+docstring const qstring_to_ucs4(QString const & qstr)
 {
 	int ls = qstr.size();
-	vector<char_type> ucs4;
+	docstring ucs4;
 	for (int i = 0; i < ls; ++i)
-		ucs4.push_back(static_cast<boost::uint32_t>(qstr[i].unicode()));
+		ucs4 += static_cast<char_type>(qstr[i].unicode());
 
 	return ucs4;
 }

src/frontends/qt4/qt_helpers.h

@@ -66,11 +66,11 @@ QString const toqstr(std::string const & str);
  *
  * QString uses ucs2 (a.k.a utf16) internally.
  */
+QString const toqstr(lyx::docstring const & ucs4);
+
 QString const ucs4_to_qstring(lyx::char_type const * str, size_t ls);
 
-QString const ucs4_to_qstring(std::vector<lyx::char_type> const & ucs4);
+lyx::docstring const qstring_to_ucs4(QString const & qstr);
-
-std::vector<lyx::char_type> qstring_to_ucs4(QString const & qstr);
 
 void qstring_to_ucs4(QString const & qstr, std::vector<lyx::char_type> & ucs4);
 

src/insets/insettabular.C

@@ -49,6 +49,8 @@
 #include <iostream>
 #include <limits>
 
+using lyx::docstring;
+
 using lyx::cap::dirtyTabularStack;
 using lyx::cap::tabularStackDirty;
 
@@ -658,7 +660,8 @@ void InsetTabular::doDispatch(LCursor & cur, FuncRequest & cmd)
 	case LFUN_FILE_INSERT_ASCII: {
 		// FIXME: We don't know the encoding of filenames
 		string const tmpstr = getContentsOfAsciiFile(&cur.bv(), lyx::to_utf8(cmd.argument()), false);
-		if (!tmpstr.empty() && !insertAsciiString(cur.bv(), tmpstr, false))
+		// FIXME: We don't know the encoding of the file
+		if (!tmpstr.empty() && !insertAsciiString(cur.bv(), lyx::from_utf8(tmpstr), false))
 			cur.undispatched();
 		break;
 	}
@@ -696,14 +699,14 @@ void InsetTabular::doDispatch(LCursor & cur, FuncRequest & cmd)
 
 	case LFUN_CLIPBOARD_PASTE:
 	case LFUN_PRIMARY_SELECTION_PASTE: {
-		string const clip = (cmd.action == LFUN_CLIPBOARD_PASTE) ?
+		docstring const clip = (cmd.action == LFUN_CLIPBOARD_PASTE) ?
 			cur.bv().owner()->gui().clipboard().get() :
 			cur.bv().owner()->gui().selection().get();
 		if (clip.empty())
 			break;
 		// pass to InsertAsciiString, but
 		// only if we have multi-cell content
-		if (clip.find_first_of("\t\n") != string::npos) {
+		if (clip.find_first_of(lyx::from_ascii("\t\n")) != docstring::npos) {
 			if (insertAsciiString(cur.bv(), clip, false)) {
 				// content has been replaced,
 				// so cursor might be invalid
@@ -1784,7 +1787,7 @@ bool InsetTabular::copySelection(LCursor & cur)
 	ostringstream os;
 	OutputParams const runparams;
 	paste_tabular->plaintext(cur.buffer(), os, runparams, 0, true, '\t');
-	cur.bv().owner()->gui().clipboard().put(os.str());
+	cur.bv().owner()->gui().clipboard().put(lyx::from_utf8(os.str()));
 	// mark tabular stack dirty
 	// FIXME: this is a workaround for bug 1919. Should be removed for 1.5,
 	// when we (hopefully) have a one-for-all paste mechanism.
@@ -1903,7 +1906,7 @@ bool InsetTabular::forceDefaultParagraphs(idx_type cell) const
 }
 
 
-bool InsetTabular::insertAsciiString(BufferView & bv, string const & buf,
+bool InsetTabular::insertAsciiString(BufferView & bv, docstring const & buf,
 				     bool usePaste)
 {
 	if (buf.length() <= 0)
@@ -1912,10 +1915,11 @@ bool InsetTabular::insertAsciiString(BufferView & bv, string const & buf,
 	col_type cols = 1;
 	row_type rows = 1;
 	col_type maxCols = 1;
-	string::size_type const len = buf.length();
+	docstring::size_type const len = buf.length();
-	string::size_type p = 0;
+	docstring::size_type p = 0;
 
-	while (p < len && (p = buf.find_first_of("\t\n", p)) != string::npos) {
+	while (p < len &&
+	       (p = buf.find_first_of(lyx::from_ascii("\t\n"), p)) != docstring::npos) {
 		switch (buf[p]) {
 		case '\t':
 			++cols;
@@ -1947,7 +1951,7 @@ bool InsetTabular::insertAsciiString(BufferView & bv, string const & buf,
 		row = tabular.row_of_cell(cell);
 	}
 
-	string::size_type op = 0;
+	docstring::size_type op = 0;
 	idx_type const cells = loctab->getNumberOfCells();
 	p = 0;
 	cols = ocol;
@@ -1955,7 +1959,7 @@ bool InsetTabular::insertAsciiString(BufferView & bv, string const & buf,
 	col_type const columns = loctab->columns();
 
 	while (cell < cells && p < len && row < rows &&
-	       (p = buf.find_first_of("\t\n", p)) != string::npos)
+	       (p = buf.find_first_of(lyx::from_ascii("\t\n"), p)) != docstring::npos)
 	{
 		if (p >= len)
 			break;

src/insets/insettabular.h

@@ -187,7 +187,7 @@ private:
 	void getSelection(LCursor & cur, row_type & rs, row_type & re,
 			  col_type & cs, col_type & ce) const;
 	///
-	bool insertAsciiString(BufferView &, std::string const & buf, bool usePaste);
+	bool insertAsciiString(BufferView &, lyx::docstring const & buf, bool usePaste);
 	/// are we operating on several cells?
 	bool tablemode(LCursor & cur) const;
 

src/insets/insettext.C

@@ -52,6 +52,7 @@
 #include <boost/bind.hpp>
 #include <boost/current_function.hpp>
 
+using lyx::docstring;
 using lyx::pos_type;
 
 using lyx::graphics::PreviewLoader;
@@ -377,7 +378,7 @@ void InsetText::markNew(bool track_changes)
 }
 
 
-void InsetText::setText(string const & data, LyXFont const & font)
+void InsetText::setText(docstring const & data, LyXFont const & font)
 {
 	clear();
 	Paragraph & first = paragraphs().front();

src/insets/insettext.h

@@ -77,7 +77,7 @@ public:
 	///
 	Code lyxCode() const { return TEXT_CODE; }
 	///
-	void setText(std::string const &, LyXFont const &);
+	void setText(lyx::docstring const &, LyXFont const &);
 	///
 	void setAutoBreakRows(bool);
 	///

src/lyx_cb.C

@@ -55,6 +55,7 @@
 #include <cerrno>
 #include <fstream>
 
+using lyx::docstring;
 using lyx::support::addName;
 using lyx::support::bformat;
 using lyx::support::destroyDir;
@@ -357,7 +358,8 @@ void insertAsciiFile(BufferView * bv, string const & f, bool asParagraph)
 	if (!bv->available())
 		return;
 
-	string const tmpstr = getContentsOfAsciiFile(bv, f, asParagraph);
+	// FIXME: We don't know the encoding of the file
+	docstring const tmpstr = lyx::from_utf8(getContentsOfAsciiFile(bv, f, asParagraph));
 	if (tmpstr.empty())
 		return;
 

src/lyxfind.C

@@ -206,7 +206,7 @@ bool stringSelected(BufferView * bv, string const & searchstr,
 	// if nothing selected or selection does not equal search
 	// string search and select next occurance and return
 	string const & str1 = searchstr;
-	string const str2 = bv->cursor().selectionAsString(false);
+	string const str2 = lyx::to_utf8(bv->cursor().selectionAsString(false));
 	if ((cs && str1 != str2) || lowercase(str1) != lowercase(str2)) {
 		find(bv, searchstr, cs, mw, fw);
 		return false;

src/lyxtext.h

@@ -260,9 +260,9 @@ public:
 	/* these things are for search and replace */
 
 	/// needed to insert the selection
-	void insertStringAsLines(LCursor & cur, std::string const & str);
+	void insertStringAsLines(LCursor & cur, lyx::docstring const & str);
 	/// needed to insert the selection
-	void insertStringAsParagraphs(LCursor & cur, std::string const & str);
+	void insertStringAsParagraphs(LCursor & cur, lyx::docstring const & str);
 
 	/// current text width
 	int width() const;

src/mathed/math_nestinset.C

@@ -1086,9 +1086,9 @@ void MathNestInset::lfunMousePress(LCursor & cur, FuncRequest & cmd)
 	} else if (cmd.button() == mouse_button::button2) {
 		MathArray ar;
 		if (cur.selection())
-			asArray(bv.cursor().selectionAsString(false), ar);
+			asArray(lyx::to_utf8(bv.cursor().selectionAsString(false)), ar);
 		else
-			asArray(bv.owner()->gui().selection().get(), ar);
+			asArray(lyx::to_utf8(bv.owner()->gui().selection().get()), ar);
 
 		cur.insert(ar);
 		bv.mouseSetCursor(cur);

src/support/lstrings.C

@@ -338,11 +338,15 @@ int tokenPos(string const & a, char delim, string const & tok)
 }
 
 
-string const subst(string const & a, char oldchar, char newchar)
+namespace {
+
+template<typename Ch> inline
+std::basic_string<Ch> const subst(std::basic_string<Ch> const & a, Ch oldchar, Ch newchar)
 {
-	string tmp(a);
+	typedef std::basic_string<Ch> String;
-	string::iterator lit = tmp.begin();
+	String tmp(a);
-	string::iterator end = tmp.end();
+	typename String::iterator lit = tmp.begin();
+	typename String::iterator end = tmp.end();
 	for (; lit != end; ++lit)
 		if ((*lit) == oldchar)
 			(*lit) = newchar;
@@ -350,13 +354,14 @@ string const subst(string const & a, char oldchar, char newchar)
 }
 
 
-string const subst(string const & a,
+template<typename String> inline
-		   string const & oldstr, string const & newstr)
+String const subst(String const & a,
+		   String const & oldstr, String const & newstr)
 {
 	BOOST_ASSERT(!oldstr.empty());
-	string lstr = a;
+	String lstr = a;
-	string::size_type i = 0;
+	typename String::size_type i = 0;
-	string::size_type const olen = oldstr.length();
+	typename String::size_type const olen = oldstr.length();
 	while ((i = lstr.find(oldstr, i)) != string::npos) {
 		lstr.replace(i, olen, newstr);
 		i += newstr.length(); // We need to be sure that we dont
@@ -365,6 +370,35 @@ string const subst(string const & a,
 	return lstr;
 }
 
+}
+
+
+string const subst(string const & a, char oldchar, char newchar)
+{
+	return subst<char>(a, oldchar, newchar);
+}
+
+
+docstring const subst(docstring const & a,
+		char_type oldchar, char_type newchar)
+{
+	return subst<char_type>(a, oldchar, newchar);
+}
+
+
+string const subst(string const & a,
+		string const & oldstr, string const & newstr)
+{
+	return subst<string>(a, oldstr, newstr);
+}
+
+
+docstring const subst(docstring const & a,
+		docstring const & oldstr, docstring const & newstr)
+{
+	return subst<docstring>(a, oldstr, newstr);
+}
+
 
 string const trim(string const & a, char const * p)
 {
@@ -546,23 +580,24 @@ int findToken(char const * const str[], string const & search_token)
 }
 
 
-string const externalLineEnding(string const & str)
+docstring const externalLineEnding(docstring const & str)
 {
 #if defined(__APPLE__)
 	// The MAC clipboard uses \r for lineendings, and we use \n
 	return subst(str, '\n', '\r');
 #elif defined (_WIN32) || (defined (__CYGWIN__) && defined (X_DISPLAY_MISSING))
 	// Windows clipboard uses \r\n for lineendings, and we use \n
-	return subst(str, "\n", "\r\n");
+	return subst(str, lyx::from_ascii("\n"), lyx::from_ascii("\r\n"));
 #else
 	return str;
 #endif
 }
 
 
-string const internalLineEnding(string const & str)
+docstring const internalLineEnding(docstring const & str)
 {
-	string s = subst(str, "\r\n", "\n");
+	docstring const s = subst(str,
+			lyx::from_ascii("\r\n"), lyx::from_ascii("\n"));
 	return subst(s, '\r', '\n');
 }
 

src/support/lstrings.h

@@ -130,10 +130,18 @@ int tokenPos(std::string const & a, char delim, std::string const & tok);
 /// Substitute all \a oldchar with \a newchar
 std::string const subst(std::string const & a, char oldchar, char newchar);
 
+/// Substitute all \a oldchar with \a newchar
+docstring const subst(docstring const & a,
+		char_type oldchar, char_type newchar);
+
 /// substitutes all instances of \a oldstr with \a newstr
 std::string const subst(std::string const & a,
 		   std::string const & oldstr, std::string const & newstr);
 
+/// substitutes all instances of \a oldstr with \a newstr
+docstring const subst(docstring const & a,
+		docstring const & oldstr, docstring const & newstr);
+
 /** Trims characters off the end and beginning of a string.
     \code
     trim("ccabccc", "c") == "ab".
@@ -188,10 +196,10 @@ std::string const getStringFromVector(std::vector<std::string> const & vec,
 int findToken(char const * const str[], std::string const & search_token);
 
 /// Convert internal line endings to line endings as expected by the OS
-std::string const externalLineEnding(std::string const & str);
+docstring const externalLineEnding(docstring const & str);
 
 /// Convert line endings in any formnat to internal line endings
-std::string const internalLineEnding(std::string const & str);
+docstring const internalLineEnding(docstring const & str);
 
 
 #ifdef I_AM_NOT_AFRAID_OF_HEADER_LIBRARIES

src/support/textutils.h

@@ -18,7 +18,7 @@
 
 /// return true if the char is a line separator
 inline
-bool isLineSeparatorChar(char c)
+bool isLineSeparatorChar(lyx::char_type c)
 {
 	return c == ' ';
 }

src/text2.C

@@ -583,10 +583,10 @@ string LyXText::getStringToIndex(LCursor const & cur)
 {
 	BOOST_ASSERT(this == cur.text());
 
-	string idxstring;
+	docstring idxstring;
-	if (cur.selection()) {
+	if (cur.selection())
 		idxstring = cur.selectionAsString(false);
-	} else {
+	else {
 		// Try implicit word selection. If there is a change
 		// in the language the implicit word selection is
 		// disabled.
@@ -601,7 +601,7 @@ string LyXText::getStringToIndex(LCursor const & cur)
 			idxstring = tmpcur.selectionAsString(false);
 	}
 
-	return idxstring;
+	return lyx::to_utf8(idxstring);
 }
 
 
@@ -647,7 +647,7 @@ void LyXText::insertInset(LCursor & cur, InsetBase * inset)
 
 
 // needed to insert the selection
-void LyXText::insertStringAsLines(LCursor & cur, string const & str)
+void LyXText::insertStringAsLines(LCursor & cur, docstring const & str)
 {
 	cur.buffer().insertStringAsLines(pars_, cur.pit(), cur.pos(),
 					 current_font, str, autoBreakRows_);
@@ -656,9 +656,9 @@ void LyXText::insertStringAsLines(LCursor & cur, string const & str)
 
 // turn double CR to single CR, others are converted into one
 // blank. Then insertStringAsLines is called
-void LyXText::insertStringAsParagraphs(LCursor & cur, string const & str)
+void LyXText::insertStringAsParagraphs(LCursor & cur, docstring const & str)
 {
-	string linestr = str;
+	docstring linestr = str;
 	bool newline_inserted = false;
 
 	for (string::size_type i = 0, siz = linestr.size(); i < siz; ++i) {

src/text3.C

@@ -144,7 +144,7 @@ namespace {
 	void mathDispatch(LCursor & cur, FuncRequest const & cmd, bool display)
 	{
 		recordUndo(cur);
-		string sel = cur.selectionAsString(false);
+		string sel = lyx::to_utf8(cur.selectionAsString(false));
 		//lyxerr << "selection is: '" << sel << "'" << endl;
 
 		// It may happen that sel is empty but there is a selection
@@ -904,7 +904,7 @@ void LyXText::dispatch(LCursor & cur, FuncRequest & cmd)
 
 	case LFUN_CLIPBOARD_PASTE: {
 		cur.clearSelection();
-		string const clip = bv->owner()->gui().clipboard().get();
+		docstring const clip = bv->owner()->gui().clipboard().get();
 		if (!clip.empty()) {
 			recordUndo(cur);
 			if (cmd.argument() == "paragraph")
@@ -917,7 +917,7 @@ void LyXText::dispatch(LCursor & cur, FuncRequest & cmd)
 
 	case LFUN_PRIMARY_SELECTION_PASTE: {
 		cur.clearSelection();
-		string const clip = bv->owner()->gui().selection().get();
+		docstring const clip = bv->owner()->gui().selection().get();
 		if (!clip.empty()) {
 			recordUndo(cur);
 			if (cmd.argument() == "paragraph")
@@ -1449,7 +1449,7 @@ void LyXText::dispatch(LCursor & cur, FuncRequest & cmd)
 	}
 
 	case LFUN_THESAURUS_ENTRY: {
-		string arg = lyx::to_utf8(cmd.argument());
+		docstring arg = cmd.argument();
 		if (arg.empty()) {
 			arg = cur.selectionAsString(false);
 			// FIXME
@@ -1459,7 +1459,7 @@ void LyXText::dispatch(LCursor & cur, FuncRequest & cmd)
 				arg = cur.selectionAsString(false);
 			}
 		}
-		bv->owner()->getDialogs().show("thesaurus", arg);
+		bv->owner()->getDialogs().show("thesaurus", lyx::to_utf8(arg));
 		break;
 	}