Implement paste from LaTeX and HTML (bug #3096)

As discussed on the list. No automatic contents detection is done, the user
needs to use the special paste menu instead. I used the new TempFile class
for safe temporary file handling.
The documentation would go into section 2.2 of UserGuide.lyx, but I am not
allowed to edit that document.

lib/ui/stdmenus.inc

@@ -156,6 +156,8 @@ Menuset
 	Menu "edit_paste"
 		Item "Plain Text|T" "clipboard-paste"
 		Item "Plain Text, Join Lines|J" "clipboard-paste paragraph"
+		Item "HTML Text|H" "paste html"
+		Item "LaTeX Text|L" "paste latex"
 		Separator
 		Item "Selection|S" "primary-selection-paste"
 		Item "Selection, Join Lines|i" "primary-selection-paste paragraph"

src/Buffer.cpp

@@ -102,6 +102,7 @@
 #include "support/Package.h"
 #include "support/PathChanger.h"
 #include "support/Systemcall.h"
+#include "support/TempFile.h"
 #include "support/textutils.h"
 #include "support/types.h"
 
@@ -978,6 +979,49 @@ bool Buffer::readDocument(Lexer & lex)
 }
 
 
+bool Buffer::importString(string const & format, docstring const & contents, ErrorList & errorList)
+{
+	Format const * fmt = formats.getFormat(format);
+	if (!fmt)
+		return false;
+	// It is important to use the correct extension here, since some
+	// converters create a wrong output file otherwise (e.g. html2latex)
+	TempFile const tempfile("Buffer_importStringXXXXXX." + fmt->extension());
+	FileName const name(tempfile.name());
+	ofdocstream os(name.toFilesystemEncoding().c_str());
+	bool const success = (os << contents);
+	os.close();
+
+	bool converted = false;
+	if (success) {
+		params().compressed = false;
+
+		// remove dummy empty par
+		paragraphs().clear();
+
+		converted = importFile(format, name, errorList);
+	}
+
+	if (name.exists())
+		name.removeFile();
+	return converted;
+}
+
+
+bool Buffer::importFile(string const & format, FileName const & name, ErrorList & errorList)
+{
+	if (!theConverters().isReachable(format, "lyx"))
+		return false;
+
+	TempFile const tempfile("Buffer_importFileXXXXXX.lyx");
+	FileName const lyx(tempfile.name());
+	if (theConverters().convert(0, name, lyx, name, format, "lyx", errorList))
+		return readFile(lyx) == ReadSuccess;
+
+	return false;
+}
+
+
 bool Buffer::readString(string const & s)
 {
 	params().compressed = false;
@@ -1125,7 +1169,7 @@ Buffer::ReadStatus Buffer::parseLyXFormat(Lexer & lex,
 Buffer::ReadStatus Buffer::convertLyXFormat(FileName const & fn,
 	FileName & tmpfile, int from_format)
 {
-	tmpfile = FileName::tempName("Buffer_convertLyXFormat");
+	tmpfile = FileName::tempName("Buffer_convertLyXFormatXXXXXX.lyx");
 	if(tmpfile.empty()) {
 		Alert::error(_("Conversion failed"),
 			bformat(_("%1$s is from a different"
@@ -2765,11 +2809,12 @@ string Buffer::absFileName() const
 
 string Buffer::filePath() const
 {
-	int last = d->filename.onlyPath().absFileName().length() - 1;
+	string const abs = d->filename.onlyPath().absFileName();
+	if (abs.empty())
+		return abs;
+	int last = abs.length() - 1;
 
-	return d->filename.onlyPath().absFileName()[last] == '/'
-		? d->filename.onlyPath().absFileName()
-		: d->filename.onlyPath().absFileName() + "/";
+	return abs[last] == '/' ? abs : abs + '/';
 }
 
 

src/Buffer.h

@@ -222,6 +222,10 @@ public:
 	/// emergency or autosave files, one should use \c loadLyXFile.
 	/// /sa loadLyXFile
 	ReadStatus loadThisLyXFile(support::FileName const & fn);
+	/// import a new document from a string
+	bool importString(std::string const &, docstring const &, ErrorList &);
+	/// import a new file
+	bool importFile(std::string const &, support::FileName const &, ErrorList &);
 	/// read a new document from a string
 	bool readString(std::string const &);
 	/// Reloads the LyX file

src/CutAndPaste.cpp

@@ -1010,16 +1010,21 @@ void pasteFromStack(Cursor & cur, ErrorList & errorList, size_t sel_index)
 }
 
 
-void pasteClipboardText(Cursor & cur, ErrorList & errorList, bool asParagraphs)
+void pasteClipboardText(Cursor & cur, ErrorList & errorList, bool asParagraphs,
+                        Clipboard::TextType type)
 {
 	// Use internal clipboard if it is the most recent one
+	// This overrides asParagraphs and type on purpose!
 	if (theClipboard().isInternal()) {
 		pasteFromStack(cur, errorList, 0);
 		return;
 	}
 
 	// First try LyX format
-	if (theClipboard().hasLyXContents()) {
+	if ((type == Clipboard::LyXTextType ||
+	     type == Clipboard::LyXOrPlainTextType ||
+	     type == Clipboard::AnyTextType) &&
+	    theClipboard().hasTextContents(Clipboard::LyXTextType)) {
 		string lyx = theClipboard().getAsLyX();
 		if (!lyx.empty()) {
 			// For some strange reason gcc 3.2 and 3.3 do not accept
@@ -1035,8 +1040,42 @@ void pasteClipboardText(Cursor & cur, ErrorList & errorList, bool asParagraphs)
 		}
 	}
 
+	// Then try TeX and HTML
+	Clipboard::TextType types[2] = {Clipboard::HtmlTextType, Clipboard::LaTeXTextType};
+	string names[2] = {"html", "latex"};
+	for (int i = 0; i < 2; ++i) {
+		if (type != types[i] && type != Clipboard::AnyTextType)
+			continue;
+		bool available = theClipboard().hasTextContents(types[i]);
+
+		// If a specific type was explicitly requested, try to
+		// interpret plain text: The user told us that the clipboard
+		// contents is in the desired format
+		if (!available && type == types[i]) {
+			types[i] = Clipboard::PlainTextType;
+			available = theClipboard().hasTextContents(types[i]);
+		}
+
+		if (available) {
+			docstring text = theClipboard().getAsText(types[i]);
+			available = !text.empty();
+			if (available) {
+				// For some strange reason gcc 3.2 and 3.3 do not accept
+				// Buffer buffer(string(), false);
+				Buffer buffer("", false);
+				buffer.setUnnamed(true);
+				if (buffer.importString(names[i], text, errorList)) {
+					cur.recordUndo();
+					pasteParagraphList(cur, buffer.paragraphs(),
+						buffer.params().documentClassPtr(), errorList);
+					return;
+				}
+			}
+		}
+	}
+
 	// Then try plain text
-	docstring const text = theClipboard().getAsText();
+	docstring const text = theClipboard().getAsText(Clipboard::PlainTextType);
 	if (text.empty())
 		return;
 	cur.recordUndo();
@@ -1065,7 +1104,7 @@ void pasteSimpleText(Cursor & cur, bool asParagraphs)
 		asParagraphs = false;
 	} else {
 		// Then try plain text
-		text = theClipboard().getAsText();
+		text = theClipboard().getAsText(Clipboard::PlainTextType);
 	}
 
 	if (text.empty())

src/CutAndPaste.h

@@ -87,8 +87,9 @@ void pasteSelection(Cursor & cur, ErrorList &);
 /// Replace the current selection with the clipboard contents as text
 /// (internal or external: which is newer).
 /// Does handle undo. Does only work in text, not mathed.
-void pasteClipboardText(Cursor & cur, ErrorList & errorList,
-	bool asParagraphs = true);
+/// \p asParagraphs is only considered if plain text is pasted.
+void pasteClipboardText(Cursor & cur, ErrorList & errorList, bool asParagraphs,
+	Clipboard::TextType preferedType = Clipboard::LyXOrPlainTextType);
 /// Replace the current selection with the clipboard contents as graphic.
 /// Does handle undo. Does only work in text, not mathed.
 void pasteClipboardGraphics(Cursor & cur, ErrorList & errorList,

src/LyXAction.cpp

@@ -1215,7 +1215,7 @@ void LyXAction::init()
  * \var lyx::FuncCode lyx::LFUN_PASTE
  * \li Action: Pastes material (text or picture) from the active clipboard.
  * \li Syntax: paste [<TYPE>|<NUM>]
- * \li Params: <TYPE>: emf|pdf|png|jpeg|linkback|wmf \n
+ * \li Params: <TYPE>: emf|pdf|png|jpeg|linkback|wmf|latex|html \n
                <NUM>: number of the selection in the internal clipboard stack to be pasted.
  * \endvar
  */

src/Text3.cpp

@@ -1248,11 +1248,15 @@ void Text::dispatch(Cursor & cur, FuncRequest & cmd)
 				     && !theClipboard().hasTextContents())
 				pasteClipboardGraphics(cur, bv->buffer().errorList("Paste"));
 			else
-				pasteClipboardText(cur, bv->buffer().errorList("Paste"));
+				pasteClipboardText(cur, bv->buffer().errorList("Paste"), true);
 		} else if (isStrUnsignedInt(arg)) {
 			// we have a numerical argument
 			pasteFromStack(cur, bv->buffer().errorList("Paste"),
 				       convert<unsigned int>(arg));
+		} else if (arg == "html" || arg == "latex") {
+			Clipboard::TextType type = (arg == "html") ?
+				Clipboard::HtmlTextType : Clipboard::LaTeXTextType;
+			pasteClipboardText(cur, bv->buffer().errorList("Paste"), true, type);
 		} else {
 			Clipboard::GraphicsType type = Clipboard::AnyGraphicsType;
 			if (arg == "pdf")
@@ -1267,7 +1271,6 @@ void Text::dispatch(Cursor & cur, FuncRequest & cmd)
 				type = Clipboard::EmfGraphicsType;
 			else if (arg == "wmf")
 				type = Clipboard::WmfGraphicsType;
-
 			else
 				LASSERT(false, /**/);
 
@@ -2772,6 +2775,20 @@ bool Text::getStatus(Cursor & cur, FuncRequest const & cmd,
 			break;
 		}
 
+		// explicit text type?
+		if (arg == "html") {
+			// Do not enable for PlainTextType, since some tidying in the
+			// frontend is needed for HTML, which is too unsafe for plain text.
+			enable = theClipboard().hasTextContents(Clipboard::HtmlTextType);
+			break;
+		} else if (arg == "latex") {
+			// LaTeX is usually not available on the clipboard with
+			// the correct MIME type, but in plain text.
+			enable = theClipboard().hasTextContents(Clipboard::PlainTextType) ||
+			         theClipboard().hasTextContents(Clipboard::LaTeXTextType);
+			break;
+		}
+
 		// explicit graphics type?
 		Clipboard::GraphicsType type = Clipboard::AnyGraphicsType;
 		if ((arg == "pdf" && (type = Clipboard::PdfGraphicsType))

src/frontends/Clipboard.h

@@ -42,6 +42,15 @@ public:
 		AnyGraphicsType
 	};
 
+	enum TextType {
+		AnyTextType,
+		LyXOrPlainTextType,
+		PlainTextType,
+		HtmlTextType,
+		LaTeXTextType,
+		LyXTextType,
+	};
+
 	/**
 	 * Get the system clipboard contents. The format is as written in
 	 * .lyx files (may even be an older version than ours if it comes
@@ -51,8 +60,8 @@ public:
 	 * clipboard.
 	 */
 	virtual std::string const getAsLyX() const = 0;
-	/// Get the contents of the window system clipboard in plain text format.
-	virtual docstring const getAsText() const = 0;
+	/// Get the contents of the window system clipboard in any text format except LyxTextType.
+	virtual docstring const getAsText(TextType type) const = 0;
 	/// Get the contents of the window system clipboard as graphics file.
 	virtual FileName getAsGraphics(Cursor const & cur, GraphicsType type) const = 0;
 
@@ -67,10 +76,8 @@ public:
 	 */
 	virtual void put(std::string const & lyx, docstring const & html, docstring const & text) = 0;
 
-	/// Does the clipboard contain LyX contents?
-	virtual bool hasLyXContents() const = 0;
 	/// Does the clipboard contain text contents?
-	virtual bool hasTextContents() const = 0;
+	virtual bool hasTextContents(TextType type = AnyTextType) const = 0;
 	/// Does the clipboard contain graphics contents of a certain type?
 	virtual bool hasGraphicsContents(GraphicsType type = AnyGraphicsType) const = 0;
 	/// state of clipboard.

src/frontends/qt4/GuiClipboard.cpp

@@ -44,6 +44,7 @@
 #include <QMimeData>
 #include <QString>
 #include <QStringList>
+#include <QTextDocument>
 
 #include <boost/crc.hpp>
 
@@ -98,6 +99,8 @@ QByteArray CacheMimeData::data(QString const & mimeType) const
 
 
 QString const lyxMimeType(){ return "application/x-lyx"; }
+QString const texMimeType(){ return "text/x-tex"; }
+QString const latexMimeType(){ return "application/x-latex"; }
 QString const pdfMimeType(){ return "application/pdf"; }
 QString const emfMimeType(){ return "image/x-emf"; }
 QString const wmfMimeType(){ return "image/x-wmf"; }
@@ -328,12 +331,80 @@ FileName GuiClipboard::getAsGraphics(Cursor const & cur, GraphicsType type) cons
 }
 
 
-docstring const GuiClipboard::getAsText() const
+namespace {
+/**
+ * Tidy up a HTML chunk coming from the clipboard.
+ * This is needed since different applications put different kinds of HTML
+ * on the clipboard:
+ * - With or without the <?xml> tag
+ * - With or without the <!DOCTYPE> tag
+ * - With or without the <html> tag
+ * - With or without the <body> tag
+ * - With or without the <p> tag
+ * Since we are going to write a HTML file for external converters we need
+ * to ensure that it is a well formed HTML file, including all the mentioned tags.
+ */
+QString tidyHtml(QString input)
+{
+	// Misuse QTextDocument to cleanup the HTML.
+	// As a side effect, all visual markup like <tt> is converted to CSS,
+	// which is ignored by gnuhtml2latex.
+	// While this may be seen as a bug by some people it is actually a
+	// good thing, since we do import structure, but ignore all visual
+	// clutter.
+	QTextDocument converter;
+	converter.setHtml(input);
+	return converter.toHtml("utf-8");
+}
+}
+
+
+docstring const GuiClipboard::getAsText(TextType type) const
 {
 	// text data from other applications
-	QString const str = qApp->clipboard()->text(QClipboard::Clipboard)
+	if ((type == AnyTextType || type == LyXOrPlainTextType) && hasTextContents(LyXTextType))
+		type = LyXTextType;
+	if (type == AnyTextType && hasTextContents(LaTeXTextType))
+		type = LaTeXTextType;
+	if (type == AnyTextType && hasTextContents(HtmlTextType))
+		type = HtmlTextType;
+	QString str;
+	switch (type) {
+	case LyXTextType:
+		// must not convert to docstring, since file can contain
+		// mixed encodings (use getAsLyX() instead)
+		break;
+	case AnyTextType:
+	case LyXOrPlainTextType:
+	case PlainTextType:
+		str = qApp->clipboard()->text(QClipboard::Clipboard)
 				.normalized(QString::NormalizationForm_C);
-	LYXERR(Debug::ACTION, "GuiClipboard::getAsText(): `" << str << "'");
+		break;
+	case LaTeXTextType: {
+		QMimeData const * source =
+			qApp->clipboard()->mimeData(QClipboard::Clipboard);
+		if (source) {
+			// First try LaTeX, then TeX (we do not distinguish
+			// for clipboard purposes)
+			if (source->hasFormat(latexMimeType())) {
+				str = source->data(latexMimeType());
+				str = str.normalized(QString::NormalizationForm_C);
+			} else if (source->hasFormat(texMimeType())) {
+				str = source->data(texMimeType());
+				str = str.normalized(QString::NormalizationForm_C);
+			}
+		}
+		break;
+	}
+	case HtmlTextType: {
+		QString subtype = "html";
+		str = qApp->clipboard()->text(subtype, QClipboard::Clipboard)
+				.normalized(QString::NormalizationForm_C);
+		str = tidyHtml(str);
+		break;
+	}
+	}
+	LYXERR(Debug::ACTION, "GuiClipboard::getAsText(" << type << "): `" << str << "'");
 	if (str.isNull())
 		return docstring();
 
@@ -369,15 +440,27 @@ void GuiClipboard::put(string const & lyx, docstring const & html, docstring con
 }
 
 
-bool GuiClipboard::hasLyXContents() const
+bool GuiClipboard::hasTextContents(Clipboard::TextType type) const
 {
-	return cache_.hasFormat(lyxMimeType());
-}
-
-
-bool GuiClipboard::hasTextContents() const
-{
-	return cache_.hasText();
+	switch (type) {
+	case AnyTextType:
+		return cache_.hasFormat(lyxMimeType()) || cache_.hasText() ||
+		       cache_.hasHtml() || cache_.hasFormat(latexMimeType()) ||
+		       cache_.hasFormat(texMimeType());
+	case LyXOrPlainTextType:
+		return cache_.hasFormat(lyxMimeType()) || cache_.hasText();
+	case LyXTextType:
+		return cache_.hasFormat(lyxMimeType());
+	case PlainTextType:
+		return cache_.hasText();       
+	case HtmlTextType:
+		return cache_.hasHtml();
+	case LaTeXTextType:
+		return cache_.hasFormat(latexMimeType()) ||
+		       cache_.hasFormat(texMimeType());
+	}
+	// shut up compiler
+	return false;
 }
 
 
@@ -425,7 +508,7 @@ bool GuiClipboard::hasGraphicsContents(Clipboard::GraphicsType type) const
 
 bool GuiClipboard::isInternal() const
 {
-	if (!hasLyXContents())
+	if (!hasTextContents(LyXTextType))
 		return false;
 
 	// ownsClipboard() is also true for stuff coming from dialogs, e.g.
@@ -473,10 +556,10 @@ void GuiClipboard::on_dataChanged()
 	for (int i = 0; i < l.count(); i++)
 		LYXERR(Debug::ACTION, l.value(i));
 
-	text_clipboard_empty_ = qApp->clipboard()->
+	plaintext_clipboard_empty_ = qApp->clipboard()->
 		text(QClipboard::Clipboard).isEmpty();
 
-	has_lyx_contents_ = hasLyXContents();
+	has_text_contents_ = hasTextContents();
 	has_graphics_contents_ = hasGraphicsContents();
 }
 
@@ -487,9 +570,9 @@ bool GuiClipboard::empty() const
 	// clipboard. The plaintext version is empty if the LyX version
 	// contains only one inset, and the LyX version is empty if the
 	// clipboard does not come from LyX.
-	if (!text_clipboard_empty_)
+	if (!plaintext_clipboard_empty_)
 		return false;
-	return !has_lyx_contents_ && !has_graphics_contents_;
+	return !has_text_contents_ && !has_graphics_contents_;
 }
 
 } // namespace frontend

src/frontends/qt4/GuiClipboard.h

@@ -69,11 +69,10 @@ public:
 	//@{
 	std::string const getAsLyX() const;
 	FileName getAsGraphics(Cursor const & cur, GraphicsType type) const;
-	docstring const getAsText() const;
+	docstring const getAsText(TextType type) const;
 	void put(std::string const & lyx, docstring const & html, docstring const & text);
-	bool hasLyXContents() const;
 	bool hasGraphicsContents(GraphicsType type = AnyGraphicsType) const;
-	bool hasTextContents() const;
+	bool hasTextContents(TextType typetype = AnyTextType) const;
 	bool isInternal() const;
 	bool hasInternal() const;
 	bool empty() const;
@@ -86,8 +85,8 @@ private Q_SLOTS:
 	void on_dataChanged();
 
 private:
-	bool text_clipboard_empty_;
-	bool has_lyx_contents_;
+	bool plaintext_clipboard_empty_;
+	bool has_text_contents_;
 	bool has_graphics_contents_;
 	/// the cached mime data used to describe the information
 	/// that can be stored in the clipboard

src/insets/InsetTabular.cpp

@@ -4329,7 +4329,7 @@ void InsetTabular::doDispatch(Cursor & cur, FuncRequest & cmd)
 	case LFUN_CLIPBOARD_PASTE:
 	case LFUN_PRIMARY_SELECTION_PASTE: {
 		docstring const clip = (act == LFUN_CLIPBOARD_PASTE) ?
-			theClipboard().getAsText() :
+			theClipboard().getAsText(Clipboard::PlainTextType) :
 			theSelection().get();
 		if (clip.empty())
 			break;

src/mathed/InsetMathGrid.cpp

@@ -1334,7 +1334,7 @@ void InsetMathGrid::doDispatch(Cursor & cur, FuncRequest & cmd)
 		cap::replaceSelection(cur);
 		docstring topaste;
 		if (cmd.argument().empty() && !theClipboard().isInternal())
-			topaste = theClipboard().getAsText();
+			topaste = theClipboard().getAsText(Clipboard::PlainTextType);
 		else {
 			idocstringstream is(cmd.argument());
 			int n = 0;

src/mathed/InsetMathNest.cpp

@@ -576,7 +576,7 @@ void InsetMathNest::doDispatch(Cursor & cur, FuncRequest & cmd)
 		replaceSelection(cur);
 		docstring topaste;
 		if (cmd.argument().empty() && !theClipboard().isInternal())
-			topaste = theClipboard().getAsText();
+			topaste = theClipboard().getAsText(Clipboard::PlainTextType);
 		else {
 			size_t n = 0;
 			idocstringstream is(cmd.argument());
@@ -1461,6 +1461,13 @@ bool InsetMathNest::getStatus(Cursor & cur, FuncRequest const & cmd,
 		flag.setEnabled(!asHullInset());
 		break;
 
+	case LFUN_PASTE: {
+		docstring const & name = cmd.argument();
+		if (name == "html" || name == "latex")
+			flag.setEnabled(false);
+		break;
+	}
+
 	default:
 		ret = false;
 		break;