Fix bug http://bugzilla.lyx.org/show_bug.cgi?id=5489 and at the same time

properly fix bugs 5216 and 5280. The best thing to do would be recognizing
at configure time a buggy iconv and #defining WORKAROUND_ICONV_BUG in
config.h, but I don't know how that could be done.


git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/branches/BRANCH_1_6_X@27620 a592a061-630c-0410-9148-cb99ea01b6c8

src/Paragraph.cpp

@@ -757,16 +757,16 @@ void Paragraph::Private::latexInset(
 	}
 
 	bool close = false;
-	odocstringstream ods;
+	odocstream::pos_type const len = os.tellp();
 
 	if (inset->forceLTR() 
 	    && running_font.isRightToLeft()
 		// ERT is an exception, it should be output with no decorations at all
 		&& inset->lyxCode() != ERT_CODE) {
 	    	if (running_font.language()->lang() == "farsi")
-			ods << "\\beginL{}";
+			os << "\\beginL{}";
 		else
-			ods << "\\L{";
+			os << "\\L{";
 		close = true;
 	}
 
@@ -785,7 +785,7 @@ void Paragraph::Private::latexInset(
 	if (open_font && inset->noFontChange()) {
 		bool closeLanguage = arabtex
 			|| basefont.isRightToLeft() == running_font.isRightToLeft();
-		unsigned int count = running_font.latexWriteEndChanges(ods,
+		unsigned int count = running_font.latexWriteEndChanges(os,
 			bparams, runparams, basefont, basefont, closeLanguage);
 		column += count;
 		// if any font properties were closed, update the running_font, 
@@ -808,7 +808,7 @@ void Paragraph::Private::latexInset(
 	int tmp;
 
 	try {
-		tmp = inset->latex(ods, runparams);
+		tmp = inset->latex(os, runparams);
 	} catch (EncodingException & e) {
 		// add location information and throw again.
 		e.par_id = id_;
@@ -818,13 +818,11 @@ void Paragraph::Private::latexInset(
 
 	if (close) {
 		if (running_font.language()->lang() == "farsi")
-				ods << "\\endL{}";
+				os << "\\endL{}";
 			else
-				ods << '}';
+				os << '}';
 	}
 
-	os << ods.str();
-
 	if (tmp) {
 		for (int j = 0; j < tmp; ++j)
 			texrow.newline();
@@ -832,7 +830,7 @@ void Paragraph::Private::latexInset(
 		texrow.start(owner_->id(), i + 1);
 		column = 0;
 	} else {
-		column += ods.str().size();
+		column += os.tellp() - len;
 	}
 
 	if (owner_->lookupChange(i).type == Change::DELETED)
@@ -2050,10 +2048,8 @@ bool Paragraph::latex(BufferParams const & bparams,
 		if (!runparams.verbatim && 
 		    runparams.encoding->package() != Encoding::none &&
 		    font.language()->encoding()->package() != Encoding::none) {
-			odocstringstream ods;
+			pair<bool, int> const enc_switch = switchEncoding(os, bparams,
-			pair<bool, int> const enc_switch = switchEncoding(ods, bparams,
 					runparams, *(font.language()->encoding()));
-			os << ods.str();
 			if (enc_switch.first) {
 				column += enc_switch.second;
 				runparams.encoding = font.language()->encoding();

src/support/docstream.cpp

@@ -84,6 +84,22 @@ protected:
 			extern_type * to, extern_type * to_end,
 			extern_type *& to_next) const
 	{
+#define WORKAROUND_ICONV_BUG 1
+#if WORKAROUND_ICONV_BUG
+		// Due to a bug in some iconv versions, when the last char in
+		// the buffer is a wide char, it gets truncated (see bugs 5216,
+		// 5280, and also 5489). As a workaround, we append a null
+		// char and then remove it from output after the conversion.
+		intern_type * from_new = 0;
+		if (*(from_end - 1) >= 0x80) {
+			size_t len = from_end - from;
+			from_new = new intern_type[len + 1];
+			memcpy(from_new, from, len * sizeof(intern_type));
+			from_new[len] = 0;
+			from_end = from_new + len + 1;
+			from = from_new;
+		}
+#endif
 		size_t inbytesleft = (from_end - from) * sizeof(intern_type);
 		size_t outbytesleft = (to_end - to) * sizeof(extern_type);
 		from_next = from;
@@ -91,6 +107,12 @@ protected:
 		result const retval = do_iconv(out_cd_,
 				reinterpret_cast<char const **>(&from_next),
 				&inbytesleft, &to_next, &outbytesleft);
+#if WORKAROUND_ICONV_BUG
+		// Remove from output the null char that we inserted at the end
+		// of the input buffer in order to circumvent an iconv bug.
+		if (from_new)
+			--to_next;
+#endif
 		if (retval == base::error) {
 			fprintf(stderr,
 				"Error %d returned from iconv when converting from %s to %s: %s\n",
@@ -119,6 +141,9 @@ protected:
 			fputc('\n', stderr);
 			fflush(stderr);
 		}
+#if WORKAROUND_ICONV_BUG
+		delete[] from_new;
+#endif
 		return retval;
 	}
 	virtual result do_unshift(state_type &, extern_type * to,