From 6c889209a77252b66f464ae075d37c7b99d2e493 Mon Sep 17 00:00:00 2001 From: Juergen Spitzmueller Date: Fri, 13 Mar 2020 11:49:07 +0100 Subject: [PATCH] Improve performance of includeonly with "maintain counters and references" First, we do not need to run bibtex/biber on the maintenance run, as the necessary references will be generated on the includeonly run. Second, exclude the master from DepTable in maintenance run, as the master is re-checked in any case in the includeonly run, and as it will always be detected as changed due to the \includeonly statement, which will trigger a complete build. More improvements to follow. --- src/Buffer.cpp | 5 +++-- src/Converter.cpp | 3 ++- src/Converter.h | 2 +- src/LaTeX.cpp | 40 ++++++++++++++++++++++++++++++---------- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/Buffer.cpp b/src/Buffer.cpp index 11241afa4e..8504c9e4e5 100644 --- a/src/Buffer.cpp +++ b/src/Buffer.cpp @@ -4502,8 +4502,9 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir string const ext = theFormats().extension(format); FileName const tmp_result_file(changeExtension(filename, ext)); Converters::RetVal const retval = - converters.convert(this, FileName(filename), tmp_result_file, - FileName(absFileName()), backend_format, format, error_list); + converters.convert(this, FileName(filename), tmp_result_file, + FileName(absFileName()), backend_format, format, + error_list, Converters::none, includeall); if (retval == Converters::KILLED) return ExportCancel; bool success = (retval == Converters::SUCCESS); diff --git a/src/Converter.cpp b/src/Converter.cpp index 49aa95b534..5ea8866ce2 100644 --- a/src/Converter.cpp +++ b/src/Converter.cpp @@ -400,7 +400,7 @@ Converters::RetVal Converters::convert(Buffer const * buffer, FileName const & from_file, FileName const & to_file, FileName const & orig_from, string const & from_format, string const & to_format, - ErrorList & errorList, int conversionflags) + ErrorList & errorList, int conversionflags, bool includeall) { if (from_format == to_format) return move(from_format, from_file, to_file, false) ? @@ -488,6 +488,7 @@ Converters::RetVal Converters::convert(Buffer const * buffer, runparams.only_childbibs = !bp.useBiblatex() && !bp.useBibtopic() && bp.multibib == "child"; + runparams.includeall = includeall; } // Some converters (e.g. lilypond) can only output files to the diff --git a/src/Converter.h b/src/Converter.h index 9a2e267896..684895e984 100644 --- a/src/Converter.h +++ b/src/Converter.h @@ -193,7 +193,7 @@ public: support::FileName const & from_file, support::FileName const & to_file, support::FileName const & orig_from, std::string const & from_format, std::string const & to_format, - ErrorList & errorList, int conversionflags = none); + ErrorList & errorList, int conversionflags = none, bool includeall = false); /// void update(Formats const & formats); /// diff --git a/src/LaTeX.cpp b/src/LaTeX.cpp index 9e08ffec47..c62535c56e 100644 --- a/src/LaTeX.cpp +++ b/src/LaTeX.cpp @@ -15,6 +15,7 @@ #include +#include "Buffer.h" #include "BufferList.h" #include "LaTeX.h" #include "LyXRC.h" @@ -226,6 +227,14 @@ int LaTeX::run(TeXErrors & terr) } if (had_depfile) { + if (runparams.includeall) { + // On an "includeall" call (whose purpose is to set up/maintain counters and references + // for includeonly), we remove the master from the dependency list since + // (1) it will be checked anyway on the subsequent includeonly run + // (2) the master is always changed (due to the \includeonly line), and this alone would + // trigger a complete, expensive run each time + head.remove_file(file); + } // Update the checksums head.update(); // Can't just check if anything has changed because it might @@ -339,7 +348,12 @@ int LaTeX::run(TeXErrors & terr) // run bibtex // if (scanres & UNDEF_CIT || scanres & RERUN || run_bibtex) - if (scanres & UNDEF_CIT || run_bibtex) { + // We do not run bibtex/biber on an "includeall" call (whose purpose is + // to set up/maintain counters and references for includeonly) since + // (1) bibliographic references will be updated on the subsequent includeonly run + // (2) this would trigger a complete run each time (as references in non-included + // children are removed on subsequent includeonly runs) + if (!runparams.includeall && (scanres & UNDEF_CIT || run_bibtex)) { // Here we must scan the .aux file and look for // "\bibdata" and/or "\bibstyle". If one of those // tags is found -> run bibtex and set rerun = true; @@ -395,7 +409,12 @@ int LaTeX::run(TeXErrors & terr) // rerun bibtex? // Complex bibliography packages such as Biblatex require // an additional bibtex cycle sometimes. - if (scanres & UNDEF_CIT) { + // We do not run bibtex/biber on an "includeall" call (whose purpose is + // to set up/maintain counters and references for includeonly) since + // (1) bibliographic references will be updated on the subsequent includeonly run + // (2) this would trigger a complete run each time (as references in non-included + // children are removed on subsequent includeonly runs) + if (!runparams.includeall && scanres & UNDEF_CIT) { // Here we must scan the .aux file and look for // "\bibdata" and/or "\bibstyle". If one of those // tags is found -> run bibtex and set rerun = true; @@ -826,8 +845,8 @@ int LaTeX::scanLogFile(TeXErrors & terr) //TODO: TL 2020 engines will contain new commandline switch --cnf-line which we //can use to set max_print_line variable for appropriate length and detect all //errors correctly. - if (contains(token, "There were undefined citations.") || - prefixIs(token, "Package biblatex Warning: The following entry could not be found")) + if (!runparams.includeall && (contains(token, "There were undefined citations.") || + prefixIs(token, "Package biblatex Warning: The following entry could not be found"))) retval |= UNDEF_CIT; if (prefixIs(token, "LaTeX Warning:") || @@ -854,7 +873,7 @@ int LaTeX::scanLogFile(TeXErrors & terr) retval |= RERUN; LYXERR(Debug::LATEX, "We should rerun."); //"Citation `cit' on page X undefined on input line X." - } else if (contains(token, "Citation") + } else if (!runparams.includeall && contains(token, "Citation") //&& contains(token, "on input line") //often split to newline && contains(token, "undefined")) { retval |= UNDEF_CIT; @@ -870,7 +889,7 @@ int LaTeX::scanLogFile(TeXErrors & terr) //If label is too long pdlaftex log line splitting will make the above fail //so we catch at least this generic statement occuring for both CIT & REF. - } else if (contains(token, "There were undefined references.")) { + } else if (!runparams.includeall && contains(token, "There were undefined references.")) { if (!(retval & UNDEF_CIT)) //if not handled already retval |= UNDEF_REF; } @@ -880,7 +899,8 @@ int LaTeX::scanLogFile(TeXErrors & terr) retval |= PACKAGE_WARNING; if (contains(token, "natbib Warning:")) { // Natbib warnings - if (contains(token, "Citation") + if (!runparams.includeall + && contains(token, "Citation") && contains(token, "on page") && contains(token, "undefined")) { retval |= UNDEF_CIT; @@ -888,9 +908,9 @@ int LaTeX::scanLogFile(TeXErrors & terr) terr.insertRef(getLineNumber(token), from_ascii("Citation undefined"), from_utf8(token), child_name); } - } else if (contains(token, "run BibTeX")) { + } else if (!runparams.includeall && contains(token, "run BibTeX")) { retval |= UNDEF_CIT; - } else if (contains(token, "run Biber")) { + } else if (!runparams.includeall && contains(token, "run Biber")) { retval |= UNDEF_CIT; biber = true; } else if (contains(token, "Rerun LaTeX") || @@ -1045,7 +1065,7 @@ int LaTeX::scanLogFile(TeXErrors & terr) retval |= TEX_WARNING; } else if (prefixIs(token, "Underfull ")) { retval |= TEX_WARNING; - } else if (contains(token, "Rerun to get citations")) { + } else if (!runparams.includeall && contains(token, "Rerun to get citations")) { // Natbib seems to use this. retval |= UNDEF_CIT; } else if (contains(token, "No pages of output")