Normalize everything that comes from 'outside' (plain text import,

keyboard input via kmap, clipboard and selection) to normalized form KC
(precomposed characters) since we don't support the decomposed form very
well.


git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@17702 a592a061-630c-0410-9148-cb99ea01b6c8

src/frontends/qt4/GuiClipboard.C

@@ -66,7 +66,8 @@ string const GuiClipboard::getAsLyX() const
 docstring const GuiClipboard::getAsText() const
 {
 	// text data from other applications
-	QString const str = qApp->clipboard()->text(QClipboard::Clipboard);
+	QString const str = qApp->clipboard()->text(QClipboard::Clipboard)
+				.normalized(QString::NormalizationForm_KC);
 	LYXERR(Debug::ACTION) << "GuiClipboard::getAsText(): `"
 	                      << fromqstr(str) << "'" << endl;
 	if (str.isNull())

src/frontends/qt4/GuiSelection.C

@@ -58,7 +58,8 @@ void GuiSelection::haveSelection(bool own)
 
 docstring const GuiSelection::get() const
 {
-	QString const str = qApp->clipboard()->text(QClipboard::Selection);
+	QString const str = qApp->clipboard()->text(QClipboard::Selection)
+				.normalized(QString::NormalizationForm_KC);
 	LYXERR(Debug::ACTION) << "GuiSelection::get: " << fromqstr(str)
 	                      << endl;
 	if (str.isNull())

src/insets/insettabular.C

@@ -701,9 +701,8 @@ void InsetTabular::doDispatch(LCursor & cur, FuncRequest & cmd)
 	case LFUN_FILE_INSERT_PLAINTEXT_PARA:
 	case LFUN_FILE_INSERT_PLAINTEXT: {
 		// FIXME UNICODE
-		string const tmpstr = getContentsOfPlaintextFile(&cur.bv(), to_utf8(cmd.argument()), false);
+		docstring const tmpstr = getContentsOfPlaintextFile(&cur.bv(), to_utf8(cmd.argument()), false);
-		// FIXME: We don't know the encoding of the file
+		if (!tmpstr.empty() && !insertPlaintextString(cur.bv(), tmpstr, false))
-		if (!tmpstr.empty() && !insertPlaintextString(cur.bv(), from_utf8(tmpstr), false))
 			cur.undispatched();
 		break;
 	}

src/lyx_cb.C

@@ -326,8 +326,7 @@ void insertPlaintextFile(BufferView * bv, string const & f, bool asParagraph)
 	if (!bv->buffer())
 		return;
 
-	// FIXME: We don't know the encoding of the file
+	docstring const tmpstr = getContentsOfPlaintextFile(bv, f, asParagraph);
-	docstring const tmpstr = from_utf8(getContentsOfPlaintextFile(bv, f, asParagraph));
 	if (tmpstr.empty())
 		return;
 
@@ -341,8 +340,8 @@ void insertPlaintextFile(BufferView * bv, string const & f, bool asParagraph)
 }
 
 
-// Read plain text file (if filename is empty, prompt for one)
+docstring const getContentsOfPlaintextFile(BufferView * bv, string const & f,
-string getContentsOfPlaintextFile(BufferView * bv, string const & f, bool asParagraph)
+		bool asParagraph)
 {
 	FileName fname(f);
 
@@ -355,12 +354,12 @@ string getContentsOfPlaintextFile(BufferView * bv, string const & f, bool asPara
 				     FileFilterList(), docstring());
 
 		if (result.first == FileDialog::Later)
-			return string();
+			return docstring();
 
 		fname = makeAbsPath(to_utf8(result.second));
 
 		if (fname.empty())
-			return string();
+			return docstring();
 	}
 
 	if (!fs::is_readable(fname.toFilesystemEncoding())) {
@@ -369,7 +368,7 @@ string getContentsOfPlaintextFile(BufferView * bv, string const & f, bool asPara
 		docstring const text = bformat(_("Could not read the specified document\n"
 							   "%1$s\ndue to the error: %2$s"), file, error);
 		Alert::error(_("Could not read file"), text);
-		return string();
+		return docstring();
 	}
 
 	ifstream ifs(fname.toFilesystemEncoding().c_str());
@@ -379,7 +378,7 @@ string getContentsOfPlaintextFile(BufferView * bv, string const & f, bool asPara
 		docstring const text = bformat(_("Could not open the specified document\n"
 							   "%1$s\ndue to the error: %2$s"), file, error);
 		Alert::error(_("Could not open file"), text);
-		return string();
+		return docstring();
 	}
 
 	ifs.unsetf(ios::skipws);
@@ -399,7 +398,8 @@ string getContentsOfPlaintextFile(BufferView * bv, string const & f, bool asPara
 	copy(ii, end, back_inserter(tmpstr));
 #endif
 
-	return tmpstr;
+	// FIXME UNICODE: We don't know the encoding of the file
+	return normalize_kc(from_utf8(tmpstr));
 }
 
 

src/lyx_cb.h

@@ -12,7 +12,7 @@
 #ifndef LYX_CB_H
 #define LYX_CB_H
 
-#include <string>
+#include "support/docstring.h"
 
 namespace lyx {
 
@@ -33,8 +33,9 @@ void autoSave(BufferView * bv);
 void newFile(BufferView * bv, std::string const & filename);
 ///
 void insertPlaintextFile(BufferView * bv, std::string const & f, bool asParagraph);
-///
+/// read plain text file (if \p f is empty, prompt for a filename)
-std::string getContentsOfPlaintextFile(BufferView * bv, std::string const & f, bool asParagraph);
+docstring const getContentsOfPlaintextFile(BufferView * bv,
+		std::string const & f, bool asParagraph);
 ///
 void reconfigure(LyXView & lv);
 

src/support/docstring.C

@@ -140,6 +140,12 @@ std::string const to_filesystem8bit(docstring const & s)
 }
 
 
+docstring const normalize_kc(docstring const & s)
+{
+	return qstring_to_ucs4(toqstr(s).normalized(QString::NormalizationForm_KC));
+}
+
+
 bool operator==(lyx::docstring const & l, char const * r)
 {
 	int const len = l.length();

src/support/docstring.h

@@ -62,6 +62,9 @@ docstring const from_filesystem8bit(std::string const & s);
 /// convert \p s from ucs4 to the encoding of the file system.
 std::string const to_filesystem8bit(docstring const & s);
 
+/// normalize \p s to precomposed form kc
+docstring const normalize_kc(docstring const & s);
+
 /// Compare a docstring with a C string of ASCII characters
 bool operator==(lyx::docstring const &, char const *);
 

src/tex-accent.C

@@ -103,8 +103,7 @@ docstring const DoAccent(docstring const & s, tex_accent accent)
 			       << lyx_accent_table[accent].name << '.' << std::endl;
 		os << s.substr(1);
 	}
-	// FIXME: We should normalize the result to precomposed form
+	return normalize_kc(os.str());
-	return os.str();
 }