Support for LaTeX mathematics in titles and abstracts

requirements.txt

@@ -36,6 +36,7 @@ django-ipware==7.0.1
 more-itertools==10.1.0
 jsmin==3.0.1
 kitchen==1.2.4
+latex2mathml==3.78.1
 libsass==0.23.0
 lxml==6.0.0
 markdown

src/core/templatetags/latex_mathml.py

@@ -0,0 +1,107 @@
+import re
+from xml.etree import ElementTree as ET
+
+import latex2mathml.converter
+import latex2mathml.exceptions
+
+from django import template
+
+register = template.Library()
+
+BLOCK_LATEX_RES = [
+    # \[...\]
+    re.compile(r"\\\[(?P<value>.+?)\\\]"),
+    # $$...$$
+    re.compile(r"\$\$(?P<value>.+?)\$\$"),
+    # \begin{displaymath}...\end{displaymath}
+    re.compile(r"\\begin\{displaymath\}(?P<value>.+?)\\end\{displaymath\}"),
+    # \begin{equation}...\end{equation}
+    re.compile(r"\\begin\{equation\}(?P<value>.+?)\\end\{equation\}"),
+]
+INLINE_LATEX_RES = [
+    # \(...\)
+    re.compile(r"\\\((?P<value>.+?)\\\)"),
+    # $...$ but not $$...$$
+    re.compile(r"(?<!\$)\$(?!\$)(?P<value>.+?)\$(?!\$)"),
+    # \begin{math}...\end{math}
+    re.compile(r"\\begin\{math\}(?P<value>.+?)\\end\{math\}"),
+]
+
+ET.register_namespace("mml", "http://www.w3.org/1998/Math/MathML")
+
+
+def _convert(value, display):
+    try:
+        return latex2mathml.converter.convert(value, display=display)
+    except Exception:
+        # The input should be returned if there is a parsing error,
+        # without delimiters to avoid MathJax attempting to parse it.
+        # We don't want MathJax touching TeX syntax because we support $...$ and
+        # MathJax does not.
+        return value
+
+
+def _to_html_block(match):
+    return _convert(match.group("value"), display="block")
+
+
+def _to_html_inline(match):
+    return _convert(match.group("value"), display="inline")
+
+
+def _add_mml_namespace_prefix(value):
+    xml = ET.fromstring(bytes(value, encoding="utf-8"))
+    return ET.tostring(xml, encoding="unicode")
+
+
+def _to_xml_block(match):
+    mathml = _convert(match.group("value"), display="block")
+    return _add_mml_namespace_prefix(mathml)
+
+
+def _to_xml_inline(match):
+    mathml = _convert(match.group("value"), display="inline")
+    return _add_mml_namespace_prefix(mathml)
+
+
+def _trim_delimiters(match):
+    return match.group("value")
+
+
+def _should_parse(journal):
+    """Whether to parse LaTeX mathematics for this journal"""
+    if journal:
+        return journal.get_setting("metadata", "latex_mathematics_title_abstract")
+    else:
+        return False
+
+
+@register.filter
+def to_mathml(value, journal, target="html", allow_block=False):
+    assert target in {"xml", "html"}
+    if _should_parse(journal):
+        if target == "html":
+            for pattern in BLOCK_LATEX_RES:
+                if allow_block:
+                    value = re.sub(pattern, _to_html_block, value)
+                else:
+                    value = re.sub(pattern, _trim_delimiters, value)
+            for pattern in INLINE_LATEX_RES:
+                value = re.sub(pattern, _to_html_inline, value)
+        elif target == "xml":
+            for pattern in BLOCK_LATEX_RES:
+                if allow_block:
+                    value = re.sub(pattern, _to_xml_block, value)
+                else:
+                    value = re.sub(pattern, _trim_delimiters, value)
+            for pattern in INLINE_LATEX_RES:
+                value = re.sub(pattern, _to_xml_inline, value)
+    return value
+
+
+@register.filter
+def strip_latex_delimiters(value, journal):
+    if _should_parse(journal):
+        for pattern in BLOCK_LATEX_RES + INLINE_LATEX_RES:
+            value = re.sub(pattern, _trim_delimiters, value)
+    return value

src/core/tests/test_templatetags.py

@@ -1,13 +1,15 @@
 from datetime import datetime, timedelta
 
+from mock import patch
 import pytz
 from django.utils import timezone
 from django.test import TestCase, override_settings
 from django.urls import set_script_prefix
 from freezegun import freeze_time
 
 from utils.testing import helpers
-from core.templatetags import fqdn, dates
+from utils import setting_handler
+from core.templatetags import fqdn, dates, latex_mathml
 
 
 class TestFqdn(TestCase):
@@ -133,3 +135,145 @@ def test_offset_date_with_default_timezone_date(self):
             input_type="date",
         )
         self.assertEqual(result, expected)
+
+
+class TestLatexMathml(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.press = helpers.create_press()
+        cls.journal, _ = helpers.create_journals()
+
+        cls.latex = r"""
+            one \[x=2\]
+            two $x=2$
+            three $$x=2$$
+            four \begin{displaymath}x=2\end{displaymath}
+            five \begin{equation}x=2\end{equation}
+            six \(x=2\)
+            seven \begin{math}x=2\end{math}
+        """
+        setting_handler.save_setting(
+            "metadata",
+            "latex_mathematics_title_abstract",
+            journal=cls.journal,
+            value="on",
+        )
+        cls.maxDiff = None
+
+    @patch("core.middleware.GlobalRequestMiddleware.get_current_request")
+    def test_to_mathml_html_allow_block(self, current_request):
+        current_request.return_value = helpers.Request(journal=self.journal)
+        expected = r"""
+            one <math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+            two <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+            three <math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+            four <math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+            five <math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+            six <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+            seven <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+        """
+        transformed = latex_mathml.to_mathml(
+            self.latex,
+            self.journal,
+            target="html",
+            allow_block=True,
+        )
+        self.assertEqual(transformed, expected)
+
+    @patch("core.middleware.GlobalRequestMiddleware.get_current_request")
+    def test_to_mathml_html_disallow_block(self, current_request):
+        current_request.return_value = helpers.Request(journal=self.journal)
+        expected = r"""
+            one x=2
+            two <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+            three x=2
+            four x=2
+            five x=2
+            six <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+            seven <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline"><mrow><mi>x</mi><mo>&#x0003D;</mo><mn>2</mn></mrow></math>
+        """
+        transformed = latex_mathml.to_mathml(
+            self.latex,
+            self.journal,
+            target="html",
+            allow_block=False,
+        )
+        self.assertEqual(transformed, expected)
+
+    @patch("core.middleware.GlobalRequestMiddleware.get_current_request")
+    def test_to_mathml_xml(self, current_request):
+        current_request.return_value = helpers.Request(journal=self.journal)
+        expected = r"""
+            one <mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" display="block"><mml:mrow><mml:mi>x</mml:mi><mml:mo>=</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:math>
+            two <mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" display="inline"><mml:mrow><mml:mi>x</mml:mi><mml:mo>=</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:math>
+            three <mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" display="block"><mml:mrow><mml:mi>x</mml:mi><mml:mo>=</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:math>
+            four <mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" display="block"><mml:mrow><mml:mi>x</mml:mi><mml:mo>=</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:math>
+            five <mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" display="block"><mml:mrow><mml:mi>x</mml:mi><mml:mo>=</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:math>
+            six <mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" display="inline"><mml:mrow><mml:mi>x</mml:mi><mml:mo>=</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:math>
+            seven <mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" display="inline"><mml:mrow><mml:mi>x</mml:mi><mml:mo>=</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:math>
+        """
+        transformed = latex_mathml.to_mathml(
+            self.latex,
+            self.journal,
+            target="xml",
+            allow_block=True,
+        )
+        self.assertEqual(transformed, expected)
+
+    @patch("core.middleware.GlobalRequestMiddleware.get_current_request")
+    def test_to_mathml_latex_syntax_error(self, current_request):
+        current_request.return_value = helpers.Request(journal=self.journal)
+        value = "some bad latex $${ \over 2}$$"
+
+        # The input should be returned if there is a parsing error,
+        # without delimiters to avoid MathJax attempting to parse it.
+        # We don't want MathJax touching TeX syntax because we support $...$ and
+        # MathJax does not.
+        expected = "some bad latex { \over 2}"
+        transformed = latex_mathml.to_mathml(value, self.journal)
+        self.assertEqual(transformed, expected)
+
+    @patch("core.middleware.GlobalRequestMiddleware.get_current_request")
+    def test_to_mathml_delimiter_syntax_error(self, current_request):
+        current_request.return_value = helpers.Request(journal=self.journal)
+        value = "bad delimiters $$x=2$ should be left alone"
+
+        # The input should be returned as is if there is a parsing error
+        expected = value
+        transformed = latex_mathml.to_mathml(value, self.journal)
+        self.assertEqual(transformed, expected)
+
+    @patch("core.middleware.GlobalRequestMiddleware.get_current_request")
+    def test_to_mathml_dollars_left_alone(self, current_request):
+        current_request.return_value = helpers.Request(journal=self.journal)
+
+        # Turn latex parsing off
+        setting_handler.save_setting(
+            "metadata",
+            "latex_mathematics_title_abstract",
+            journal=self.journal,
+            value="",
+        )
+
+        value = "article about $dollars not math just $dollars"
+        expected = value
+        result = latex_mathml.to_mathml(value, self.journal)
+        self.assertEqual(result, expected)
+
+    @patch("core.middleware.GlobalRequestMiddleware.get_current_request")
+    def test_strip_delimiters(self, current_request):
+        current_request.return_value = helpers.Request(journal=self.journal)
+        expected = r"""
+            one x=2
+            two x=2
+            three x=2
+            four x=2
+            five x=2
+            six x=2
+            seven x=2
+        """
+        transformed = latex_mathml.strip_latex_delimiters(
+            self.latex,
+            self.journal,
+        )
+        self.assertEqual(transformed, expected)

src/identifiers/logic.py

@@ -15,7 +15,7 @@
 from django.urls import reverse
 from django.template.loader import render_to_string
 from django.utils.http import urlencode
-from django.utils.html import strip_tags
+from django.utils.html import mark_safe
 from django.conf import settings
 from django.contrib import messages
 from django.utils import timezone
@@ -30,6 +30,7 @@
 from identifiers import models
 from submission import models as submission_models
 
+
 logger = get_logger(__name__)
 
 CROSSREF_TIMEOUT_SECONDS = 30
@@ -380,17 +381,13 @@ def create_crossref_journal_context(
 def create_crossref_article_context(article, identifier=None):
     template_context = {
         "id": article.pk,
-        "title": "{0}{1}{2}".format(
-            article.title,
-            " " if article.subtitle is not None else "",
-            article.subtitle if article.subtitle is not None else "",
-        ),
+        "stripped_title": article.stripped_title,
         "doi": identifier.identifier
         if identifier
         else render_doi_from_pattern(article),
         "url": article.url,
         "authors": article.frozenauthor_set.all(),
-        "abstract": strip_tags(article.abstract or ""),
+        "safe_abstract_jats": article.safe_abstract_jats,
         "date_accepted": article.date_accepted,
         "date_published": article.date_published,
         "license": article.license.url if article.license else "",

src/identifiers/tests/test_logic.py

@@ -231,8 +231,8 @@ def test_create_crossref_journal_context(self):
     def test_create_crossref_article_context_published(self):
         self.maxDiff = None
         expected_data = {
-            "title": self.article_published.title,
-            "abstract": "",
+            "stripped_title": self.article_published.stripped_title,
+            "safe_abstract_jats": self.article_published.safe_abstract_jats,
             "url": self.article_published.url,
             "authors": [
                 author.email for author in self.article_published.frozenauthor_set.all()
@@ -257,8 +257,8 @@ def test_create_crossref_article_context_published(self):
 
     def test_create_crossref_article_context_not_published(self):
         expected_data = {
-            "title": self.article_one.title,
-            "abstract": self.article_one.abstract,
+            "title": self.article_one.stripped_title,
+            "abstract": self.article_one.safe_abstract_jats,
             "url": self.article_one.url,
             "authors": [
                 author.email for author in self.article_one.frozenauthor_set.all()

src/review/logic.py

@@ -36,6 +36,7 @@
     email,
     models as core_models,
 )
+from core.templatetags.latex_mathml import to_mathml
 from review import models
 from review.const import EditorialDecisions as ED
 from events import logic as event_logic
@@ -234,11 +235,12 @@ def get_article_details_for_review(article):
         <b>Section</b>: {section}<br />
         <b>Keywords</b>: {keywords}<br />
         <b>Abstract</b>:<br />
-            {article.abstract}<br />
+            {abstract}<br />
         """.format(
         article=article,
         section=article.section.name if article.section else None,
         keywords=", ".join(kw.word for kw in article.keywords.all()),
+        abstract=article.safe_abstract_html,
     )
     return mark_safe(detail_string)
 

src/rss/views.py

@@ -6,7 +6,7 @@
 from django.contrib.syndication.views import Feed
 from django.urls import reverse
 from django.utils import timezone
-from django.template.defaultfilters import striptags
+from django.template.defaultfilters import striptags, truncatewords
 from django.contrib.contenttypes.models import ContentType
 
 from comms import models as comms_models
@@ -83,10 +83,10 @@ def items(self, obj):
             ).order_by("-date_published")[:10]
 
     def item_title(self, item):
-        return striptags(item.title)
+        return item.stripped_title
 
     def item_description(self, item):
-        return truncatesmart(item.abstract, 400)
+        return truncatewords(item.stripped_abstract, 400)
 
     def item_author_name(self, item):
         if hasattr(item, "posted_by"):