Fix bug #5238: LyX 1.6 fails with non-ascii chars in path (on Windows)

git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@33524 a592a061-630c-0410-9148-cb99ea01b6c8

src/Buffer.cpp

@@ -930,7 +930,7 @@ Buffer::ReadStatus Buffer::readFile(Lexer & lex, FileName const & filename,
 			<< ' ' << quoteName(lyx2lyx.toFilesystemEncoding())
 			<< " -t " << convert<string>(LYX_FORMAT)
 			<< " -o " << quoteName(tmpfile.toFilesystemEncoding())
-			<< ' ' << quoteName(filename.toFilesystemEncoding());
+			<< ' ' << quoteName(filename.toSafeFilesystemEncoding());
 		string const command_str = command.str();
 
 		LYXERR(Debug::INFO, "Running '" << command_str << '\'');
@@ -977,8 +977,6 @@ bool Buffer::save() const
 	// We don't need autosaves in the immediate future. (Asger)
 	resetAutosaveTimers();
 
-	string const encodedFilename = d->filename.toFilesystemEncoding();
-
 	FileName backupName;
 	bool madeBackup = false;
 
@@ -1037,10 +1035,10 @@ bool Buffer::writeFile(FileName const & fname) const
 	message(str);
 
 	if (params().compressed) {
-		gz::ogzstream ofs(fname.toFilesystemEncoding().c_str(), ios::out|ios::trunc);
+		gz::ogzstream ofs(fname.toSafeFilesystemEncoding().c_str(), ios::out|ios::trunc);
 		retval = ofs && write(ofs);
 	} else {
-		ofstream ofs(fname.toFilesystemEncoding().c_str(), ios::out|ios::trunc);
+		ofstream ofs(fname.toSafeFilesystemEncoding().c_str(), ios::out|ios::trunc);
 		retval = ofs && write(ofs);
 	}
 

src/Lexer.cpp

@@ -264,7 +264,7 @@ bool Lexer::Pimpl::setFile(FileName const & filename)
 			LYXERR(Debug::LYXLEX, "Error in Lexer::setFile: "
 				"file or stream already set.");
 		}
-		fb_.open(filename.toFilesystemEncoding().c_str(), ios::in);
+		fb_.open(filename.toSafeFilesystemEncoding().c_str(), ios::in);
 		is.rdbuf(&fb_);
 		name = filename.absFilename();
 		lineno = 0;

src/graphics/GraphicsConverter.cpp

@@ -208,8 +208,8 @@ static string const move_file(string const & from_file, string const & to_file)
 		return string();
 
 	ostringstream command;
-	command << "fromfile = utf8ToDefaultEncoding(" << from_file << ")\n"
+	command << "fromfile = toUnicode(" << from_file << ")\n"
-		<< "tofile = utf8ToDefaultEncoding("   << to_file << ")\n\n"
+		<< "tofile = toUnicode("   << to_file << ")\n\n"
 		<< "try:\n"
 		<< "  os.rename(fromfile, tofile)\n"
 		<< "except:\n"
@@ -264,20 +264,17 @@ static void build_script(FileName const & from_file,
 
 	script << "#!/usr/bin/env python\n"
 		  "# -*- coding: utf-8 -*-\n"
-		  "import os, shutil, sys, locale\n\n"
+		  "import os, shutil, sys\n\n"
 		  "def unlinkNoThrow(file):\n"
 		  "  ''' remove a file, do not throw if an error occurs '''\n"
 		  "  try:\n"
 		  "    os.unlink(file)\n"
 		  "  except:\n"
 		  "    pass\n\n"
-		  "def utf8ToDefaultEncoding(file):\n"
+		  "def toUnicode(file):\n"
-		  "  ''' if possible, convert to the default encoding '''\n"
+		  "  ''' if possible, convert to python unicode format '''\n"
 		  "  try:\n"
-		  "    language, output_encoding = locale.getdefaultlocale()\n"
+		  "    return unicode(file, 'utf8')\n"
-		  "    if output_encoding == None:\n"
-		  "      output_encoding = 'latin1'\n"
-		  "    return unicode(file, 'utf8').encode(output_encoding)\n"
 		  "  except:\n"
 		  "    return file\n\n";
 
@@ -301,10 +298,10 @@ static void build_script(FileName const & from_file,
 	// in python, but the converters might be shell scripts and have more
 	// troubles with it.
 	string outfile = addExtension(to_base.absFilename(), getExtension(from_file.absFilename()));
-	script << "infile = utf8ToDefaultEncoding("
+	script << "infile = toUnicode("
 			<< quoteName(from_file.absFilename(), quote_python)
 			<< ")\n"
-		  "outfile = utf8ToDefaultEncoding("
+		  "outfile = toUnicode("
 			<< quoteName(outfile, quote_python) << ")\n"
 		  "shutil.copy(infile, outfile)\n";
 
@@ -313,7 +310,7 @@ static void build_script(FileName const & from_file,
 	// This has the added benefit that all other files that may be
 	// generated by the converter are deleted when LyX closes and do not
 	// clutter the real working directory.
-	script << "os.chdir(utf8ToDefaultEncoding("
+	script << "os.chdir(toUnicode("
 	       << quoteName(onlyPath(outfile)) << "))\n";
 
 	if (edgepath.empty()) {
@@ -321,7 +318,7 @@ static void build_script(FileName const & from_file,
 		// converter path from from_format to to_format, so we use
 		// the default converter.
 		script << "infile = outfile\n"
-		       << "outfile = utf8ToDefaultEncoding("
+		       << "outfile = toUnicode("
 		       << quoteName(to_file, quote_python) << ")\n";
 
 		ostringstream os;
@@ -363,11 +360,11 @@ static void build_script(FileName const & from_file,
 		outfile = addExtension(to_base.absFilename(), conv.To->extension());
 
 		// Store these names in the python script
-		script << "infile = utf8ToDefaultEncoding("
+		script << "infile = toUnicode("
 				<< quoteName(infile, quote_python) << ")\n"
-			  "infile_base = utf8ToDefaultEncoding("
+			  "infile_base = toUnicode("
 				<< quoteName(infile_base, quote_python) << ")\n"
-			  "outfile = utf8ToDefaultEncoding("
+			  "outfile = toUnicode("
 				<< quoteName(outfile, quote_python) << ")\n"
 			  "outdir  = os.path.dirname(outfile)\n" ;
 

src/support/FileName.cpp

@@ -66,6 +66,7 @@
 #endif // SUM_WITH_MMAP
 
 using namespace std;
+using namespace lyx::support;
 
 // OK, this is ugly, but it is the only workaround I found to compile
 // with gcc (any version) on a system which uses a non-GNU toolchain.
@@ -190,7 +191,7 @@ string FileName::absFilename() const
 
 string FileName::realPath() const
 {
-	return os::real_path(toFilesystemEncoding());
+	return os::real_path(absFilename());
 }
 
 
@@ -270,13 +271,21 @@ bool FileName::changePermission(unsigned long int mode) const
 
 string FileName::toFilesystemEncoding() const
 {
-	// FIXME: This doesn't work on Windows for non ascii file names with Qt < 4.4.
+	// This doesn't work on Windows for non ascii file names.
-	// Provided that Windows package uses Qt4.4, this isn't a problem.
 	QByteArray const encoded = QFile::encodeName(d->fi.absoluteFilePath());
 	return string(encoded.begin(), encoded.end());
 }
 
 
+string FileName::toSafeFilesystemEncoding() const
+{
+	// This will work on Windows for non ascii file names.
+	QString const safe_path = toqstr(os::safe_internal_path(absFilename()));
+	QByteArray const encoded = QFile::encodeName(safe_path);
+	return string(encoded.begin(), encoded.end());
+}
+
+
 FileName FileName::fromFilesystemEncoding(string const & name)
 {
 	QByteArray const encoded(name.c_str(), name.length());

src/support/FileName.h

@@ -72,10 +72,24 @@ public:
 
 	/**
 	 * Get the file name in the encoding used by the file system.
-	 * Only use this for accessing the file, e.g. with an fstream.
+	 * Only use this for passing file names to external commands.
+	 * Warning: On Windows this is not unicode safe and should not
+	 * be used for accessing files with an fstream, for example.
 	 */
 	std::string toFilesystemEncoding() const;
 
+	/**
+	 * Get the file name in a unicode safe encoding used by the file system.
+	 * Only use this for accessing the file with standard I/O functions
+	 * non explicitly unicode aware, e.g. with an fstream. This can also
+	 * be used for passing file names to external commands, but only if
+	 * you are sure that the stem of the name will not be used for
+	 * producing derivative files. For example, don't use this for passing
+	 * file names to LaTeX, as the stem of the .dvi file will not correspond
+	 * to the stem of the .tex file anymore.
+	 */
+	std::string toSafeFilesystemEncoding() const;
+
 	/// returns true if the file exists
 	bool exists() const;
 	/// refreshes the file info

src/support/os.h

@@ -73,6 +73,13 @@ std::string external_path(std::string const & p);
 /// \p p and the return value are encoded in utf8.
 std::string internal_path(std::string const & p);
 
+/// Converts a host OS style path to a unicode safe unix style.
+/// On Windows, this is achieved by using the short form of the path,
+/// which can be safely passed to standard I/O functions expecting narrow
+/// char paths even when the path contains non-ascii chars.
+/// \p p and the return value are encoded in utf8.
+std::string safe_internal_path(std::string const & p);
+
 /// Converts a unix style path list to host OS style.
 /// \p p and the return value are encoded in utf8.
 std::string external_path_list(std::string const & p);
@@ -129,8 +136,7 @@ bool canAutoOpenFile(std::string const & ext, auto_open_mode const mode = VIEW);
 bool autoOpenFile(std::string const & filename, auto_open_mode const mode = VIEW);
 
 /** Resolves a path such that it does not contain '.', '..', or symbolic links.
-  * \warning the path must already be in the filesystem encoding.
+  * \p path and the return value are encoded in utf8.
-  * \returns the resolved path in utf8 encoding.
   */
 std::string real_path(std::string const & path);
 

src/support/os_cygwin.cpp

@@ -290,6 +290,12 @@ string internal_path(string const & p)
 }
 
 
+string safe_internal_path(string const & p)
+{
+	return convert_path(p, PathStyle(posix));
+}
+
+
 string external_path_list(string const & p)
 {
 	return convert_path_list(p, PathStyle(posix));

src/support/os_unix.cpp

@@ -130,6 +130,12 @@ string internal_path(string const & p)
 }
 
 
+string safe_internal_path(string const & p)
+{
+	return p;
+}
+
+
 string external_path_list(string const & p)
 {
 	return p;

src/support/os_win32.cpp

@@ -23,12 +23,15 @@
 #include "support/filetools.h"
 #include "support/lstrings.h"
 #include "support/ExceptionMessage.h"
+#include "support/qstring_helpers.h"
 
 #include "support/lassert.h"
 
 #include <cstdlib>
 #include <vector>
 
+#include <QString>
+
 /* The GetLongPathName macro may be defined on the compiling machine,
  * but we must use a bit of trickery if the resulting executable is
  * to run on a Win95 machine.
@@ -278,29 +281,51 @@ string external_path(string const & p)
 }
 
 
-static string const get_long_path(string const & short_path)
+static QString const get_long_path(QString const & short_path)
 {
-	// GetLongPathName needs the path in file system encoding.
+	// GetLongPathNameW needs the path in utf16 encoding.
-	// We can use to_local8bit, since file system encoding and the
+	vector<wchar_t> long_path(MAX_PATH);
-	// local 8 bit encoding are identical on windows.
+	DWORD result = GetLongPathNameW((wchar_t *) short_path.utf16(),
-	vector<char> long_path(MAX_PATH);
-	DWORD result = GetLongPathName(to_local8bit(from_utf8(short_path)).c_str(),
 				       &long_path[0], long_path.size());
 
 	if (result > long_path.size()) {
 		long_path.resize(result);
-		result = GetLongPathName(short_path.c_str(),
+		result = GetLongPathNameW((wchar_t *) short_path.utf16(),
 					 &long_path[0], long_path.size());
 		LASSERT(result <= long_path.size(), /**/);
 	}
 
-	return (result == 0) ? short_path : to_utf8(from_filesystem8bit(&long_path[0]));
+	return (result == 0) ? short_path : QString::fromWCharArray(&long_path[0]);
+}
+
+
+static QString const get_short_path(QString const & long_path)
+{
+	// GetShortPathNameW needs the path in utf16 encoding.
+	vector<wchar_t> short_path(MAX_PATH);
+	DWORD result = GetShortPathNameW((wchar_t *) long_path.utf16(),
+				       &short_path[0], short_path.size());
+
+	if (result > short_path.size()) {
+		short_path.resize(result);
+		result = GetShortPathNameW((wchar_t *) long_path.utf16(),
+					 &short_path[0], short_path.size());
+		LASSERT(result <= short_path.size(), /**/);
+	}
+
+	return (result == 0) ? long_path : QString::fromWCharArray(&short_path[0]);
 }
 
 
 string internal_path(string const & p)
 {
-	return subst(get_long_path(p), "\\", "/");
+	return subst(fromqstr(get_long_path(toqstr(p))), "\\", "/");
+}
+
+
+string safe_internal_path(string const & p)
+{
+	return subst(fromqstr(get_short_path(toqstr(p))), "\\", "/");
 }
 
 
@@ -482,12 +507,13 @@ bool autoOpenFile(string const & filename, auto_open_mode const mode)
 string real_path(string const & path)
 {
 	// See http://msdn.microsoft.com/en-us/library/aa366789(VS.85).aspx
-	HANDLE hpath = CreateFile(subst(path, '/', '\\').c_str(), GENERIC_READ,
+	QString qpath = get_long_path(toqstr(path));
+	HANDLE hpath = CreateFileW((wchar_t *) qpath.utf16(), GENERIC_READ,
 				FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);
 
 	if (hpath == INVALID_HANDLE_VALUE) {
 		// The file cannot be accessed.
-		return FileName::fromFilesystemEncoding(path).absFilename();
+		return path;
 	}
 
 	// Get the file size.
@@ -497,7 +523,7 @@ string real_path(string const & path)
 	if (size_lo == 0 && size_hi == 0) {
 		// A zero-length file cannot be mapped.
 		CloseHandle(hpath);
-		return FileName::fromFilesystemEncoding(path).absFilename();
+		return path;
 	}
 
 	// Create a file mapping object.
@@ -505,7 +531,7 @@ string real_path(string const & path)
 
 	if (!hmap) {
 		CloseHandle(hpath);
-		return FileName::fromFilesystemEncoding(path).absFilename();
+		return path;
 	}
 
 	// Create a file mapping to get the file name.
@@ -514,7 +540,7 @@ string real_path(string const & path)
 	if (!pmem) {
 		CloseHandle(hmap);
 		CloseHandle(hpath);
-		return FileName::fromFilesystemEncoding(path).absFilename();
+		return path;
 	}
 
 	TCHAR realpath[MAX_PATH + 1];
@@ -523,7 +549,7 @@ string real_path(string const & path)
 		UnmapViewOfFile(pmem);
 		CloseHandle(hmap);
 		CloseHandle(hpath);
-		return FileName::fromFilesystemEncoding(path).absFilename();
+		return path;
 	}
 
 	// Translate device name to UNC prefix or drive letters.