diff --git a/src/sbml/html2md/html2md.cpp b/src/sbml/html2md/html2md.cpp index 00f2420b4b..99e2bb7029 100644 --- a/src/sbml/html2md/html2md.cpp +++ b/src/sbml/html2md/html2md.cpp @@ -253,19 +253,30 @@ bool Converter::ok() const { !is_in_tag_ && index_blockquote == 0 && index_li == 0; } +// isspace()/isblank() are locale-dependent for byte values above 127. +// UTF-8 continuation bytes fall in that range, so check ASCII +// whitespace explicitly to avoid splitting a multi-byte character. +static bool IsAsciiSpace(unsigned char ch) { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\v' || ch == '\f' || ch == '\r'; +} + +static bool IsAsciiBlank(unsigned char ch) { + return ch == ' ' || ch == '\t'; +} + void Converter::LTrim(string *s) { (*s).erase((*s).begin(), find_if((*s).begin(), (*s).end(), - [](unsigned char ch) { return !std::isspace(ch); })); + [](unsigned char ch) { return !IsAsciiSpace(ch); })); } Converter *Converter::RTrim(string *s, bool trim_only_blank) { (*s).erase(find_if((*s).rbegin(), (*s).rend(), [trim_only_blank](unsigned char ch) { if (trim_only_blank) - return !isblank(ch); + return !IsAsciiBlank(ch); - return !isspace(ch); + return !IsAsciiSpace(ch); }) .base(), (*s).end()); diff --git a/src/sbml/maddy/VERSION.txt b/src/sbml/maddy/VERSION.txt index 81e5c5c2aa..4d506a2ba2 100644 --- a/src/sbml/maddy/VERSION.txt +++ b/src/sbml/maddy/VERSION.txt @@ -1,3 +1,3 @@ From https://github.com/luciansmith/maddy -Commit f3573472ba41df0ed2854550a56d1ebf9bf70b3d (branch make-tables-conformant, merged to master), based on upstream version 1.5.0. \ No newline at end of file +Changes all proposed to the maddy core source. \ No newline at end of file diff --git a/src/sbml/maddy/codespanutils.h b/src/sbml/maddy/codespanutils.h new file mode 100644 index 0000000000..a4fdd6587b --- /dev/null +++ b/src/sbml/maddy/codespanutils.h @@ -0,0 +1,139 @@ +/* + * This project is licensed under the MIT license. For more information see the + * LICENSE file. + */ +#pragma once + +// ----------------------------------------------------------------------------- + +#include +#include +#include +#include +#include + +// ----------------------------------------------------------------------------- + +namespace maddy { + +// ----------------------------------------------------------------------------- + +/** + * FindProtectedSpans + * + * Finds non-overlapping [start, end) ranges of a line that inline + * delimiter parsers (strong, emphasized, strikethrough, italic) must not + * alter: a `...` HTML span, or a code span delimited by a + * run of N backtick characters and a later run of exactly N backticks + * (per CommonMark, a run with no matching close is ordinary text, and + * scanning resumes right after it). + * + * @method + * @param {const std::string&} line + * @return {std::vector>} + */ +inline std::vector> FindProtectedSpans( + const std::string& line +) +{ + static const std::string openCodeTag = ""; + static const std::string closeCodeTag = ""; + + std::vector> spans; + std::size_t i = 0; + + while (i < line.size()) + { + if (line.compare(i, openCodeTag.size(), openCodeTag) == 0) + { + std::size_t closeStart = line.find(closeCodeTag, i + openCodeTag.size()); + if (closeStart != std::string::npos) + { + std::size_t end = closeStart + closeCodeTag.size(); + spans.emplace_back(i, end); + i = end; + continue; + } + } + + if (line[i] == '`') + { + std::size_t runStart = i; + while (i < line.size() && line[i] == '`') + { + ++i; + } + std::size_t runLength = i - runStart; + + std::size_t searchPos = i; + while (searchPos < line.size()) + { + std::size_t closeStart = line.find('`', searchPos); + if (closeStart == std::string::npos) + { + break; + } + + std::size_t closeEnd = closeStart; + while (closeEnd < line.size() && line[closeEnd] == '`') + { + ++closeEnd; + } + + if (closeEnd - closeStart == runLength) + { + spans.emplace_back(runStart, closeEnd); + i = closeEnd; + break; + } + + searchPos = closeEnd; + } + + continue; + } + + ++i; + } + + return spans; +} + +/** + * ApplyOutsideProtectedSpans + * + * Runs `transform` on each stretch of `line` that falls outside its + * protected spans (see FindProtectedSpans), leaving the spans themselves + * untouched, then writes the reassembled result back into `line`. + * + * @method + * @param {std::string&} line + * @param {const std::function&} transform + * @return {void} + */ +inline void ApplyOutsideProtectedSpans( + std::string& line, const std::function& transform +) +{ + std::string result; + std::size_t pos = 0; + + for (const auto& span : FindProtectedSpans(line)) + { + std::string segment = line.substr(pos, span.first - pos); + transform(segment); + result += segment; + result += line.substr(span.first, span.second - span.first); + pos = span.second; + } + + std::string tail = line.substr(pos); + transform(tail); + result += tail; + + line = result; +} + +// ----------------------------------------------------------------------------- + +} // namespace maddy diff --git a/src/sbml/maddy/emphasizedparser.h b/src/sbml/maddy/emphasizedparser.h index 377e6748f5..4743624227 100644 --- a/src/sbml/maddy/emphasizedparser.h +++ b/src/sbml/maddy/emphasizedparser.h @@ -9,6 +9,7 @@ #include #include +#include "maddy/codespanutils.h" #include "maddy/lineparser.h" // ----------------------------------------------------------------------------- @@ -43,12 +44,14 @@ class EmphasizedParser : public LineParser // The leading and trailing `(_*)` groups absorb any leftover underscores // from an unbalanced run (e.g. `__foo_` or `_foo____`), re-emitted // outside the tag instead of into its content. - static std::regex re( - R"((?!.*`.*|.*.*)\b(_*)_(?![\s_])(?!.*`.*|.*<\/code>.*)(.*?[^\s])_(_*)\b(?!.*`.*|.*<\/code>.*))" - ); + static std::regex re(R"(\b(_*)_(?![\s_])(.*?[^\s])_(_*)\b)"); static std::string replacement = "$1$2$3"; - line = std::regex_replace(line, re, replacement); + ApplyOutsideProtectedSpans( + line, + [](std::string& segment) + { segment = std::regex_replace(segment, re, replacement); } + ); } }; // class EmphasizedParser diff --git a/src/sbml/maddy/italicparser.h b/src/sbml/maddy/italicparser.h index 3f7075d57d..1b21e0305b 100644 --- a/src/sbml/maddy/italicparser.h +++ b/src/sbml/maddy/italicparser.h @@ -9,6 +9,7 @@ #include #include +#include "maddy/codespanutils.h" #include "maddy/lineparser.h" // ----------------------------------------------------------------------------- @@ -38,11 +39,14 @@ class ItalicParser : public LineParser */ void Parse(std::string& line) override { - static std::regex re( - R"((?!.*`.*|.*.*)\*(?!.*`.*|.*<\/code>.*)([^\*]*)\*(?!.*`.*|.*<\/code>.*))" - ); + static std::regex re(R"(\*([^\*]*)\*)"); static std::string replacement = "$1"; - line = std::regex_replace(line, re, replacement); + + ApplyOutsideProtectedSpans( + line, + [](std::string& segment) + { segment = std::regex_replace(segment, re, replacement); } + ); } }; // class ItalicParser diff --git a/src/sbml/maddy/linkparser.h b/src/sbml/maddy/linkparser.h index d555d40ba2..1270029849 100644 --- a/src/sbml/maddy/linkparser.h +++ b/src/sbml/maddy/linkparser.h @@ -41,14 +41,14 @@ class LinkParser : public LineParser void Parse(std::string& line) override { // Match [name](http:://link "title text") - // NOTE: the 'no quote' bit at the beginning (^") is a hack for now: - // there should eventually be something that replaces it with '%22'. - static std::regex re(R"(\[([^\]]*)\]\( *([^)^ ^"]*) *\"([^\"]*)\" *\))"); + // NOTE: quotes are excluded from the URL for now as a hack; there + // should eventually be something that replaces them with '%22'. + static std::regex re(R"(\[([^\]]*)\]\( *([^) "]*) *\"([^\"]*)\" *\))"); static std::string replacement = "$1"; line = std::regex_replace(line, re, replacement); // Match [name](http:://link) - static std::regex re2(R"(\[([^\]]*)\]\( *([^)^ ^"]*) *\))"); + static std::regex re2(R"(\[([^\]]*)\]\( *([^) "]*) *\))"); static std::string replacement2 = "$1"; line = std::regex_replace(line, re2, replacement2); } diff --git a/src/sbml/maddy/strikethroughparser.h b/src/sbml/maddy/strikethroughparser.h index effa5d8236..df18a9c170 100644 --- a/src/sbml/maddy/strikethroughparser.h +++ b/src/sbml/maddy/strikethroughparser.h @@ -9,6 +9,7 @@ #include #include +#include "maddy/codespanutils.h" #include "maddy/lineparser.h" // ----------------------------------------------------------------------------- @@ -38,12 +39,14 @@ class StrikeThroughParser : public LineParser */ void Parse(std::string& line) override { - static std::regex re( - R"((?!.*`.*|.*.*)\~\~(?!.*`.*|.*<\/code>.*)([^\~]*)\~\~(?!.*`.*|.*<\/code>.*))" - ); + static std::regex re(R"(\~\~([^\~]*)\~\~)"); static std::string replacement = "$1"; - line = std::regex_replace(line, re, replacement); + ApplyOutsideProtectedSpans( + line, + [](std::string& segment) + { segment = std::regex_replace(segment, re, replacement); } + ); } }; // class StrikeThroughParser diff --git a/src/sbml/maddy/strongparser.h b/src/sbml/maddy/strongparser.h index b6c0de7b52..5e39223944 100644 --- a/src/sbml/maddy/strongparser.h +++ b/src/sbml/maddy/strongparser.h @@ -9,6 +9,7 @@ #include #include +#include "maddy/codespanutils.h" #include "maddy/lineparser.h" // ----------------------------------------------------------------------------- @@ -43,18 +44,23 @@ class StrongParser : public LineParser // `*` is not a word character, so `\b` next to it does not mean // "edge of a delimiter run" the way it does for `_`; the asterisk // variant is left without a word-boundary anchor. - static std::regex reAsterisk{ - R"((?!.*`.*|.*.*)\*\*(?![\s])(?!.*`.*|.*<\/code>.*)(.*?[^\s])\*\*(?!.*`.*|.*<\/code>.*))" - }; + static std::regex reAsterisk{R"(\*\*(?![\s])(.*?[^\s])\*\*)"}; // The leading and trailing `(_*)` groups absorb any leftover underscores // from an unbalanced run on either side (e.g. `___text__` or // `__text_______`), re-emitted outside the tag by the caller // instead of being swallowed into its content. - static std::regex reUnderscore{ - R"((?!.*`.*|.*.*)\b(_*)__(?![\s_])(?!.*`.*|.*<\/code>.*)(.*?[^\s])__(_*)\b(?!.*`.*|.*<\/code>.*))" - }; - line = std::regex_replace(line, reAsterisk, "$1"); - line = std::regex_replace(line, reUnderscore, "$1$2$3"); + static std::regex reUnderscore{R"(\b(_*)__(?![\s_])(.*?[^\s])__(_*)\b)"}; + + ApplyOutsideProtectedSpans( + line, + [](std::string& segment) + { + segment = + std::regex_replace(segment, reAsterisk, "$1"); + segment = + std::regex_replace(segment, reUnderscore, "$1$2$3"); + } + ); } }; // class StrongParser diff --git a/src/sbml/test/TestSBase.cpp b/src/sbml/test/TestSBase.cpp index b74ac132be..650d2c844e 100644 --- a/src/sbml/test/TestSBase.cpp +++ b/src/sbml/test/TestSBase.cpp @@ -705,6 +705,41 @@ START_TEST(test_SBase_notesMarkdown_table) END_TEST +START_TEST(test_SBase_notesMarkdown_denseLinks) +{ + /* A single notes line carrying several Markdown links back to back, + * with no line break between them (from BIOMD0000000012 -- see + * antimony issue #177), drove std::regex past its complexity limit + * under MSVC's stricter implementation. */ + Model* m = new(std::nothrow) Model(3, 1); + std::string notes = "( [KISAO_0000035](http://identifiers.org/biomodels.kisao/KISAO_0000035 \"Access to: KISAO_0000035\") ) with continuous variables ( [KISAO_0000018](http://identifiers.org/biomodels.kisao/KISAO_0000018 \"Access to: KISAO_0000018\") ). One sample algorithm to use is the CVODE solver ( [KISAO_0000019](http://identifiers.org/biomodels.kisao/KISAO_0000019 \"Access to: KISAO_0000019\") ). Second, one could simulate the system using Gillespie's direct method ( [KISAO_0000029](http://identifiers.org/biomodels.kisao/KISAO_0000029 \"Access to: KISAO_0000029\") ), which is a stochastic method ( [KISAO_0000036](http://identifiers.org/biomodels.kisao/KISAO_0000036 \"Access to: KISAO_0000036\") ) supporting adaptive timesteps ( [KISAO_0000041](http://identifiers.org/biomodels.kisao/KISAO_0000041 \"Access to: KISAO_0000041\") ) and using discrete variables ( [KISAO_0000016](http://identifiers.org/biomodels.kisao/KISAO_0000016 \"Access to: KISAO_0000016\") ). "; + + fail_unless(m->setNotesFromMarkdown(notes) == LIBSBML_OPERATION_SUCCESS); + + std::string html = m->getNotesString(); + fail_unless(html.find("KISAO_0000035") != std::string::npos); + fail_unless(html.find("KISAO_0000016") != std::string::npos); +} +END_TEST + + +START_TEST(test_SBase_notesMarkdown_trailingNonBreakingSpace) +{ + /* A non-breaking space (U+00A0, UTF-8 bytes 0xC2 0xA0) landing at + * the end of a paragraph -- as seen in several curated BioModels + * notes -- used to get its trailing byte stripped by html2md's + * line-trimming, leaving an orphaned lead byte and invalid UTF-8. */ + Model* m = new(std::nothrow) Model(3, 1); + std::string notes = "

Some text\xC2\xA0

"; + + fail_unless(m->setNotes(notes, false) == LIBSBML_OPERATION_SUCCESS); + + std::string md = m->getNotesMarkdown(); + fail_unless(md.find("Some text\xC2\xA0") != std::string::npos); +} +END_TEST + + START_TEST(test_SBase_setAnnotationString) { const char * annotation = "This is a test note"; @@ -2729,6 +2764,8 @@ create_suite_SBase (void) tcase_add_test(tcase, test_SBase_setNotesFromMarkdown2); tcase_add_test(tcase, test_SBase_setNotesFromMarkdown3); tcase_add_test(tcase, test_SBase_notesMarkdown_table); + tcase_add_test(tcase, test_SBase_notesMarkdown_denseLinks); + tcase_add_test(tcase, test_SBase_notesMarkdown_trailingNonBreakingSpace); tcase_add_test(tcase, test_SBase_setAnnotationString); tcase_add_test(tcase, test_SBase_unsetAnnotationWithCVTerms ); tcase_add_test(tcase, test_SBase_unsetAnnotationWithModelHistory );