Handle URL escaping in beamer frames

File format change -- this introduces the first LyX 2.5 format and
hence sets up the lyx2lyx machinery

lib/configure.py

@@ -877,7 +877,8 @@ def checkFormatEntries(dtl_tools):
 \Format lyx20x     20.lyx "LyX 2.0.x"             "" ""	""	"document"	""
 \Format lyx21x     21.lyx "LyX 2.1.x"             "" ""	""	"document"	""
 \Format lyx22x     22.lyx "LyX 2.2.x"             "" ""	""	"document"	""
-\Format lyx23x     23.lyx "LyX 2.3.x"             "" ""	""	"document,menu=export"	""
+\Format lyx23x     23.lyx "LyX 2.3.x"             "" ""	""	"document"	""
+\Format lyx24x     24.lyx "LyX 2.4.x"             "" ""	""	"document,menu=export"	""
 \Format clyx       cjklyx "CJK LyX 1.4.x (big5)"  "" ""	""	"document"	""
 \Format jlyx       cjklyx "CJK LyX 1.4.x (euc-jp)" "" ""	""	"document"	""
 \Format klyx       cjklyx "CJK LyX 1.4.x (euc-kr)" "" ""	""	"document"	""
@@ -1417,6 +1418,7 @@ def checkConverterEntries():
 \converter lyx        lyx21x     "$${python} $$s/lyx2lyx/lyx2lyx -V 2.1 -o $$o $$i"	""
 \converter lyx        lyx22x     "$${python} $$s/lyx2lyx/lyx2lyx -V 2.2 -o $$o $$i"	""
 \converter lyx        lyx23x     "$${python} $$s/lyx2lyx/lyx2lyx -V 2.3 -o $$o $$i"	""
+\converter lyx        lyx24x     "$${python} $$s/lyx2lyx/lyx2lyx -V 2.4 -o $$o $$i"	""
 \converter lyx        clyx       "$${python} $$s/lyx2lyx/lyx2lyx -V 1.4 -o $$o -c big5   $$i"	""
 \converter lyx        jlyx       "$${python} $$s/lyx2lyx/lyx2lyx -V 1.4 -o $$o -c euc_jp $$i"	""
 \converter lyx        klyx       "$${python} $$s/lyx2lyx/lyx2lyx -V 1.4 -o $$o -c euc_kr $$i"	""

lib/doc/Customization.lyx

@@ -15119,6 +15119,8 @@ status collapsed
 
 \change_inserted -712698321 1715492433
 -1
+\change_unchanged
+
 \end_layout
 
 \end_inset
@@ -15165,6 +15167,8 @@ status collapsed
 
 \change_inserted -712698321 1715492466
 1
+\change_unchanged
+
 \end_layout
 
 \end_inset
@@ -15201,6 +15205,8 @@ status collapsed
 
 \change_inserted -712698321 1715492536
 0
+\change_unchanged
+
 \end_layout
 
 \end_inset
@@ -15216,6 +15222,8 @@ status collapsed
 
 \backslash
 cprotect
+\change_unchanged
+
 \end_layout
 
 \end_inset
@@ -15229,6 +15237,8 @@ status collapsed
 
 \change_inserted -712698321 1715492585
 -1
+\change_unchanged
+
 \end_layout
 
 \end_inset
@@ -15243,6 +15253,8 @@ status collapsed
 
 \backslash
 cprotect
+\change_unchanged
+
 \end_layout
 
 \end_inset
@@ -15456,6 +15468,58 @@ Standard
  (So this will not affect the display of non-default paragraphs.)
 \end_layout
 
+\begin_layout Description
+
+\change_inserted -712698321 1715509676
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -712698321 1715509206
+NoPassThruChars
+\change_unchanged
+
+\end_layout
+
+\end_inset
+
+ [
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -712698321 1715509203
+string
+\change_unchanged
+
+\end_layout
+
+\end_inset
+
+]
+\family typewriter
+ 
+\family default
+Defines individual characters that should 
+\emph on
+not
+\emph default
+ be output in raw form (without special translations that \SpecialChar LaTeX
+ would require),
+ even if such handling is requested by an element in this layout.
+ This tag can be used to overwrite 
+\begin_inset Quotes els
+\end_inset
+
+PassThru
+\begin_inset Quotes ers
+\end_inset
+
+ conditions if they are unsuitable in a given context.
+\end_layout
+
 \begin_layout Description
 \begin_inset Flex Code
 status collapsed
@@ -21026,6 +21090,58 @@ InsetLayout
 \end_inset
 
 .
+\change_inserted -712698321 1715512954
+
+\end_layout
+
+\begin_layout Description
+
+\change_inserted -712698321 1715512954
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -712698321 1715512954
+NoPassThruChars
+\end_layout
+
+\end_inset
+
+ [
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+
+\change_inserted -712698321 1715512954
+string
+\end_layout
+
+\end_inset
+
+]
+\family typewriter
+ 
+\family default
+Defines individual characters that should 
+\emph on
+not
+\emph default
+ be output in raw form (without special translations that \SpecialChar LaTeX
+ would require),
+ even if such handling is requested by an element in this layout.
+ This tag can be used to overwrite 
+\begin_inset Quotes els
+\end_inset
+
+PassThru
+\begin_inset Quotes ers
+\end_inset
+
+ conditions if they are unsuitable in a given context.
+\change_unchanged
+
 \end_layout
 
 \begin_layout Description

lib/doc/de/Customization.lyx

@@ -13808,7 +13808,7 @@ cprotect
 \family sans
 cprotect
 \family default
-) gechützt werden,
+) geschützt werden,
  falls nötig.
  Damit wird die Verwendung (mancher) Verbatim-Dinge in Makros ermöglicht.
  In der Voreinstellung (Wert 
@@ -14121,6 +14121,56 @@ Standard
 \begin_inset Flex Code
 status collapsed
 
+\begin_layout Plain Layout
+NoPassThruChars
+\end_layout
+
+\end_inset
+
+ [
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+string
+\end_layout
+
+\end_inset
+
+]
+\family typewriter
+ 
+\family default
+Definiert Zeichen,
+ die 
+\emph on
+nicht
+\emph default
+ in jedem Fall unverändert ausgegeben werden sollen (d.
+\begin_inset space \thinspace{}
+\end_inset
+
+h.,
+ die ggf.
+ in einen \SpecialChar LaTeX
+-Befehl übersetzt werden),
+ auch wenn ein eingebettetes Element in diesem Layout eine unveränderte Ausgabe verlangt.
+ Damit lassen sich 
+\begin_inset Quotes gls
+\end_inset
+
+PassThru
+\begin_inset Quotes grs
+\end_inset
+
+-Bedingungen überschreiben,
+ wenn diese in einem bestimmten Umfeld unerwünscht sind.
+\end_layout
+
+\begin_layout Description
+\begin_inset Flex Code
+status collapsed
+
 \begin_layout Plain Layout
 ObsoletedBy
 \end_layout
@@ -19758,6 +19808,56 @@ InsetLayout
 \begin_inset Flex Code
 status collapsed
 
+\begin_layout Plain Layout
+NoPassThruChars
+\end_layout
+
+\end_inset
+
+ [
+\begin_inset Flex Code
+status collapsed
+
+\begin_layout Plain Layout
+string
+\end_layout
+
+\end_inset
+
+]
+\family typewriter
+ 
+\family default
+Definiert Zeichen,
+ die 
+\emph on
+nicht
+\emph default
+ in jedem Fall unverändert ausgegeben werden sollen (d.
+\begin_inset space \thinspace{}
+\end_inset
+
+h.,
+ die ggf.
+ in einen \SpecialChar LaTeX
+-Befehl übersetzt werden),
+ auch wenn ein eingebettetes Element in diesem Layout eine unveränderte Ausgabe verlangt.
+ Damit lassen sich 
+\begin_inset Quotes gls
+\end_inset
+
+PassThru
+\begin_inset Quotes grs
+\end_inset
+
+-Bedingungen überschreiben,
+ wenn diese in einem bestimmten Umfeld unerwünscht sind.
+\end_layout
+
+\begin_layout Description
+\begin_inset Flex Code
+status collapsed
+
 \begin_layout Plain Layout
 ObsoletedBy
 \end_layout

lib/layouts/beamer.layout

@@ -554,6 +554,7 @@ Style Frame
 	Example,Examples,Fact,Lemma,Proof,Theorem,LyX-Code
   EndAutoNests
   NeedCProtect    -1
+  NoPassThruChars  "%#"
 End
 
 Style PlainFrame
@@ -577,6 +578,7 @@ Style FragileFrame
     PresetArg       "fragile"
   EndArgument
   NeedCProtect      0
+  NoPassThruChars   ""
 End
 
 Style AgainFrame

lib/lyx2lyx/LyX.py

@@ -1,6 +1,6 @@
 # This file is part of lyx2lyx
 # -*- coding: utf-8 -*-
-# Copyright (C) 2002-2018 The LyX Team
+# Copyright (C) 2002-2024 The LyX Team
 # Copyright (C) 2002-2004 Dekel Tsur <dekel@lyx.org>
 # Copyright (C) 2002-2006 José Matos <jamatos@lyx.org>
 #
@@ -36,7 +36,7 @@ try:
     version__ = lyx2lyx_version.version
     stable_version = True
 except: # we are running from build directory so assume the last version
-    version__ = '2.4'
+    version__ = '2.5'
     stable_version = False
 
 default_debug__ = 2
@@ -95,8 +95,9 @@ format_relation = [("0_06",    [200], minor_versions("0.6" , 4)),
                    ("2_0", list(range(346,414)), minor_versions("2.0" , 8)),
                    ("2_1", list(range(414,475)), minor_versions("2.1" , 5)),
                    ("2_2", list(range(475,509)), minor_versions("2.2" , 4)),
-                   ("2_3", list(range(509,545)), minor_versions("2.3" , 0)),
+                   ("2_3", list(range(509,545)), minor_versions("2.3" , 7)),
-                   ("2_4", (), minor_versions("2.4" , 0))
+                   ("2_4", list(range(545,621)), minor_versions("2.4" , 0)),
+                   ("2_5", (), minor_versions("2.5" , 0))
                   ]
 
 ####################################################################

lib/lyx2lyx/Makefile.am

@@ -35,6 +35,7 @@ dist_lyx2lyx_PYTHON = \
 	lyx_2_2.py \
 	lyx_2_3.py \
 	lyx_2_4.py \
+	lyx_2_5.py \
 	profiling.py \
 	test_parser_tools.py
 

lib/lyx2lyx/lyx_2_5.py

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+# This file is part of lyx2lyx
+# Copyright (C) 2024 The LyX team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+
+""" Convert files to the file format generated by lyx 2.5"""
+
+import re, string
+import unicodedata
+import sys, os
+
+from datetime import (datetime, date, time)
+
+# Uncomment only what you need to import, please.
+
+from parser_tools import (find_end_of_inset, find_end_of_layout, find_token, find_re)
+#    count_pars_in_inset, del_complete_lines, del_token, find_end_of,
+#    find_token_backwards, find_token_exact, get_bool_value,
+#    get_containing_inset, get_containing_layout, get_option_value, get_value,
+#    get_quoted_value, is_in_inset,
+#    del_value, 
+#    find_complete_lines,
+#    find_re, find_substring,
+#    set_bool_value
+#    find_tokens, check_token
+
+#from lyx2lyx_tools import (put_cmd_in_ert, add_to_preamble, insert_to_preamble, lyx2latex,
+#                           revert_language, revert_flex_inset, str2bool)
+#  revert_font_attrs, latex_length
+#  get_ert, lyx2verbatim, length_in_bp, convert_info_insets
+#  revert_flex_inset, hex2ratio
+
+####################################################################
+# Private helper functions
+
+
+
+###############################################################################
+###
+### Conversion and reversion routines
+###
+###############################################################################
+
+def convert_url_escapes(document):
+    """Unescape # and % in URLs in frames."""
+    if document.textclass not in ['beamer', 'scrarticle-beamer', 'beamerposter', 'article-beamer']:
+        return
+
+    rurl = re.compile(r'^[%#].*')
+    i = 0
+    while True:
+        i = find_token(document.body, "\\begin_layout Frame", i + 1)
+        if i == -1:
+            return
+        j = find_end_of_layout(document.body, i)
+        if j == -1:
+            document.warning("Malformed LyX document: Could not find end of Frame layout.")
+            continue
+        k = find_token(document.body, "\\begin_inset Flex URL", i, j)
+        if k == -1:
+            continue
+        l = find_end_of_inset(document.body, k)
+        if l == -1:
+            document.warning("Malformed LyX document: Could not find end of URL inset.")
+            continue
+        while True:
+            surl = find_re(document.body, rurl, k, l)
+            if surl == -1:
+                break
+            if document.body[surl - 1] == "\\backslash":
+                del document.body[surl - 1]
+            k = surl
+        
+
+def revert_url_escapes(document):
+    """Unescape # and % in URLs in frames."""
+    if document.textclass not in ['beamer', 'scrarticle-beamer', 'beamerposter', 'article-beamer']:
+        return
+
+    rurl = re.compile(r'^(.*)([%#].*)')
+    i = 0
+    while True:
+        i = find_token(document.body, "\\begin_layout Frame", i + 1)
+        if i == -1:
+            return
+        j = find_end_of_layout(document.body, i)
+        if j == -1:
+            document.warning("Malformed LyX document: Could not find end of Frame layout.")
+            continue
+        k = find_token(document.body, "\\begin_inset Flex URL", i, j)
+        if k == -1:
+            continue
+        l = find_end_of_inset(document.body, k)
+        if l == -1:
+            document.warning("Malformed LyX document: Could not find end of URL inset.")
+            continue
+        while True:
+            surl = find_re(document.body, rurl, k, l)
+            if surl == -1:
+                break
+            m = rurl.match(document.body[surl])
+            if m:
+                if m.group(1) == "" and document.body[surl - 1] == "\\backslash":
+                    break
+                document.body[surl : surl + 1] = [m.group(1), "\\backslash", m.group(2)]
+            k = surl
+
+##
+# Conversion hub
+#
+
+supported_versions = ["2.5.0", "2.5"]
+convert = [
+           [621, [convert_url_escapes]]
+          ]
+
+
+revert =  [[620, [revert_url_escapes]]
+          ]
+
+
+if __name__ == "__main__":
+    pass

src/Layout.cpp

@@ -52,6 +52,7 @@ enum LayoutTags {
 	LT_FREE_SPACING,
 	LT_PASS_THRU,
 	LT_PASS_THRU_CHARS,
+	LT_NO_PASS_THRU_CHARS,
 	LT_PARBREAK_IS_NEWLINE,
 	LT_ITEMCOMMAND,
 	LT_ITEMSEP,
@@ -317,6 +318,7 @@ bool Layout::readIgnoreForcelocal(Lexer & lex, TextClass const & tclass,
 		{ "needprotect",    LT_NEED_PROTECT },
 		{ "newline",        LT_NEWLINE },
 		{ "nextnoindent",   LT_NEXTNOINDENT },
+		{ "nopassthruchars", LT_NO_PASS_THRU_CHARS },
 		{ "obsoletedby",    LT_OBSOLETEDBY },
 		{ "paragraphgroup", LT_PAR_GROUP },
 		{ "parbreakisnewline", LT_PARBREAK_IS_NEWLINE },
@@ -669,6 +671,10 @@ bool Layout::readIgnoreForcelocal(Lexer & lex, TextClass const & tclass,
 			lex >> pass_thru_chars;
 			break;
 
+		case LT_NO_PASS_THRU_CHARS:
+			lex >> no_pass_thru_chars;
+			break;
+
 		case LT_PARBREAK_IS_NEWLINE:
 			lex >> parbreak_is_newline;
 			break;

src/Layout.h

@@ -387,6 +387,9 @@ public:
 	bool pass_thru;
 	/// Individual chars to be passed verbatim
 	docstring pass_thru_chars;
+	/// Individual characters that must not be
+	/// passed verbatim even if normally requested
+	docstring no_pass_thru_chars;
 	///
 	bool parbreak_is_newline;
 	/// show this in toc

src/OutputParams.h

@@ -377,6 +377,10 @@ public:
 	/// Should we output verbatim specific chars?
 	docstring pass_thru_chars;
 
+	/// Do not output verbatim specific chars even
+	/// if normally requested
+	docstring no_pass_thru_chars;
+
 	/// A specific newline macro
 	std::string newlinecmd;
 

src/Paragraph.cpp

@@ -1240,9 +1240,10 @@ void Paragraph::Private::latexSpecialChar(otexstream & os,
 {
 	char_type const c = owner_->getUChar(bparams, runparams, i);
 
-	if (style.pass_thru || runparams.pass_thru || runparams.find_effective()
+	if ((style.pass_thru || runparams.pass_thru || runparams.find_effective()
 	    || contains(style.pass_thru_chars, c)
-	    || contains(runparams.pass_thru_chars, c)) {
+	    || contains(runparams.pass_thru_chars, c))
+	    && !contains(runparams.no_pass_thru_chars, c)) {
 		if (runparams.find_effective()) {
 			switch (c) {
 			case '\\':

src/insets/InsetLayout.cpp

@@ -136,6 +136,7 @@ bool InsetLayout::read(Lexer & lex, TextClass const & tclass,
 		IL_NEEDMBOXPROTECT,
 		IL_NEEDPROTECT,
 		IL_NEWLINE_CMD,
+		IL_NO_PASSTHRU_CHARS,
 		IL_PASSTHRU,
 		IL_PASSTHRU_CHARS,
 		IL_PARBREAKIGNORED,
@@ -225,6 +226,7 @@ bool InsetLayout::read(Lexer & lex, TextClass const & tclass,
 		{ "needmboxprotect", IL_NEEDMBOXPROTECT },
 		{ "needprotect", IL_NEEDPROTECT },
 		{ "newlinecmd", IL_NEWLINE_CMD },
+		{ "nopassthruchars", IL_NO_PASSTHRU_CHARS },
 		{ "obsoletedby", IL_OBSOLETEDBY },
 		{ "parbreakignored", IL_PARBREAKIGNORED },
 		{ "parbreakisnewline", IL_PARBREAKISNEWLINE },
@@ -382,6 +384,9 @@ bool InsetLayout::read(Lexer & lex, TextClass const & tclass,
 		case IL_PASSTHRU_CHARS:
 			lex >> passthru_chars_;
 			break;
+		case IL_NO_PASSTHRU_CHARS:
+			lex >> no_passthru_chars_;
+			break;
 		case IL_NEWLINE_CMD:
 			lex >> newline_cmd_;
 			break;

src/insets/InsetLayout.h

@@ -212,6 +212,8 @@ public:
 	///
 	docstring passThruChars() const { return passthru_chars_; }
 	///
+	docstring noPassThruChars() const { return no_passthru_chars_; }
+	///
 	std::string newlineCmd() const { return newline_cmd_; }
 	///
 	bool parbreakIsNewline() const { return parbreakisnewline_; }
@@ -397,6 +399,10 @@ private:
 	///
 	docstring passthru_chars_;
 	///
+	docstring no_passthru_chars_;
+	///
+	docstring no_hr_passthru_chars_;
+	///
 	std::string newline_cmd_;
 	///
 	bool parbreakisnewline_ = false;

src/insets/InsetText.cpp

@@ -549,6 +549,8 @@ void InsetText::latex(otexstream & os, OutputParams const & runparams) const
 		++rp.inulemcmd;
 	if (!il.passThruChars().empty())
 		rp.pass_thru_chars += il.passThruChars();
+	if (!il.noPassThruChars().empty())
+		rp.no_pass_thru_chars += il.noPassThruChars();
 	if (!il.newlineCmd().empty())
 		rp.newlinecmd = il.newlineCmd();
 	rp.par_begin = 0;

src/output_latex.cpp

@@ -417,6 +417,7 @@ void TeXEnvironment(Buffer const & buf, Text const & text,
 
 	OutputParams runparams = runparams_in;
 	runparams.no_cprotect = current_layout.nocprotect;
+	runparams.no_pass_thru_chars = current_layout.no_pass_thru_chars;
 
 	// This is for debugging purpose at the end.
 	pit_type const par_begin = pit;

src/version.h

@@ -32,8 +32,8 @@ extern char const * const lyx_version_info;
 
 // Do not remove the comment below, so we get merge conflict in
 // independent branches. Instead add your own.
-#define LYX_FORMAT_LYX 620 // spitz: default box frame color
+#define LYX_FORMAT_LYX 621 // spitz: handle #% in frame urls
-#define LYX_FORMAT_TEX2LYX 620
+#define LYX_FORMAT_TEX2LYX 621
 
 #if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX
 #ifndef _MSC_VER