@@ -33,7 +33,7 @@ def _compile_charset(charset, flags, code):
3333 emit = code .append
3434 for op , av in charset :
3535 emit (op )
36- if op is NEGATE :
36+ if op in ( NEGATE , INVERT ) :
3737 pass
3838 elif op is LITERAL :
3939 emit (av )
@@ -58,31 +58,40 @@ def _compile_charset(charset, flags, code):
5858def _optimize_charset (charset , iscased = None , fixup = None , fixes = None ):
5959 # internal: optimize character set.
6060 #
61- # The engine's charset() walk toggles polarity on every NEGATE (see
62- # Modules/_sre/sre_lib.h), so NEGATE markers split the set into
63- # alternating-polarity segments: a leading NEGATE is a complemented class
64- # [^...], an interior one is set difference (RL1.3). Each segment is a
65- # plain union, optimized on its own with the NEGATE boundaries kept in place.
66- negates = [ i for i , ( op , _av ) in enumerate ( charset ) if op is NEGATE ]
67- if not negates or negates == [0 ]:
61+ # The engine's charset() walk toggles its return polarity on every NEGATE
62+ # and its membership-test direction on every INVERT (see sre_lib.h) --
63+ # together they express complement, difference and intersection (RL1.3).
64+ # Each toggle-delimited segment is optimized on its own.
65+ bounds = [ i for i , ( op , _av ) in enumerate ( charset )
66+ if op in ( NEGATE , INVERT ) ]
67+ if not bounds or ( bounds == [0 ] and charset [ 0 ][ 0 ] is NEGATE ) :
6868 # Fast path: a plain union, optionally complemented as a whole -- every
6969 # charset the parser produces today, optimized as before.
7070 return _optimize_charset_segment (charset , iscased , fixup , fixes )
7171
72- # Optimize each NEGATE -delimited run on its own. _allow_anyall is off: the
72+ # Optimize each toggle -delimited run on its own. _allow_anyall is off: the
7373 # [\s\S] -> ANY_ALL / [^\s\S] -> empty shortcuts rewrite a whole set and
74- # would inject or drop a NEGATE mid-segment.
74+ # would inject or drop a toggle mid-segment.
7575 out = []
7676 hascased = False
7777 start = 0
78- for i in negates + [len (charset )]:
78+ inv = False
79+ for i in bounds + [len (charset )]:
7980 if i > start : # skip an empty run (e.g. a leading NEGATE)
80- opt , cased = _optimize_charset_segment (
81- charset [start :i ], iscased , fixup , fixes , _allow_anyall = False )
82- out .extend (opt )
83- hascased |= cased
81+ if inv :
82+ # An INVERT run intersects its members: re-emit them as
83+ # they are (already optimized, see _single_member).
84+ out .extend (charset [start :i ])
85+ else :
86+ opt , cased = _optimize_charset_segment (
87+ charset [start :i ], iscased , fixup , fixes , _allow_anyall = False )
88+ out .extend (opt )
89+ hascased |= cased
8490 if i < len (charset ):
85- out .append ((NEGATE , None ))
91+ tok = charset [i ]
92+ out .append (tok ) # re-emit the toggle in place
93+ if tok [0 ] is INVERT :
94+ inv = not inv
8695 start = i + 1
8796 return out , hascased
8897
@@ -467,43 +476,75 @@ def _fuse_branch(av):
467476 items += cs
468477 return items if tail is None else items + tail
469478
470- def _fuse_difference (data ):
471- # Replace <flat charset A> (?<![B1]) (?<![B2]) ... with the single charset
472- # [NEGATE] B1 B2 ... [NEGATE] A. Each negative lookbehind over a flat
473- # charset subtracts its set from the character A matches.
479+ def _single_member (operand ):
480+ # Reduce a flat-charset operand to one member opcode (a lone bitmap,
481+ # range or category), or None. An intersection operand must be a single
482+ # member, because each member under INVERT is a separate test.
483+ items = _parser ._flat_items (operand )
484+ if items is None :
485+ return None
486+ opt , _hascased = _optimize_charset (items )
487+ return opt if len (opt ) == 1 else None
488+
489+ def _fuse_setops (data , flags ):
490+ # Fuse <flat charset A> (?<![B]) (?<=[C]) ... -- a left-associative
491+ # chain of set difference ([A--B]) and intersection ([A&&C]) -- into the
492+ # single charset [NEGATE] <fail items> [NEGATE] [INVERT] A. The chain
493+ # is a pure conjunction, so each lookbehind appends one fail item:
494+ # "ch in B" for a difference, "ch not in C" (under INVERT) for an
495+ # intersection. Not fused under IGNORECASE, where case folding could
496+ # split a single-member intersection operand (see _single_member).
474497 out = []
475- head = None # _flat_items(A) for the fused difference now at out[-1]
476- subtrahend = None # its accumulated B items, or None when not fusing
498+ head = None # _flat_items(A) for the fused set operation now at out[-1]
499+ fails = None # its accumulated fail items, or None when not fusing
500+ inv = 0 # the INVERT state at the end of fails
477501 for op , av in data :
478- if op is ASSERT_NOT and av [0 ] < 0 : # a negative lookbehind
479- b = _parser ._flat_items (av [1 ].data )
502+ if op in (ASSERT , ASSERT_NOT ) and av [0 ] < 0 : # a lookbehind
503+ if op is ASSERT_NOT : # -- difference
504+ b = _parser ._flat_items (av [1 ].data )
505+ tinv = 0
506+ elif not flags & SRE_FLAG_IGNORECASE : # && intersection
507+ b = _single_member (av [1 ].data )
508+ tinv = 1
509+ else :
510+ b = None
480511 if b is not None :
481- if subtrahend is None and out :
512+ if fails is None and out :
482513 # the first lookbehind of a run: only now is it worth
483514 # checking whether the preceding item A is a flat charset.
484515 head = _parser ._flat_items ([out [- 1 ]])
485516 if head is not None :
486- subtrahend = []
487- if subtrahend is not None :
488- subtrahend += b
489- out [- 1 ] = (IN , [(NEGATE , None )] + subtrahend
490- + [(NEGATE , None )] + head )
517+ fails = []
518+ inv = 0
519+ if fails is not None :
520+ if inv != tinv :
521+ fails .append ((INVERT , None ))
522+ inv ^= 1
523+ fails += b
524+ tail = [(NEGATE , None )]
525+ if inv :
526+ tail .append ((INVERT , None ))
527+ out [- 1 ] = (IN , [(NEGATE , None )] + fails + tail + head )
491528 continue
492- head = subtrahend = None
529+ head = fails = None
493530 out .append ((op , av ))
494531 data [:] = out
495532
496- def _walk (seq ):
533+ def _walk (seq , flags ):
497534 for i , (op , av ) in enumerate (seq ):
498- for sub in _subpatterns (op , av ):
499- _walk (sub .data )
535+ if op is SUBPATTERN :
536+ # A group can change the flag context, e.g. (?i:...).
537+ _walk (av [3 ].data , _combine_flags (flags , av [1 ], av [2 ]))
538+ else :
539+ for sub in _subpatterns (op , av ):
540+ _walk (sub .data , flags )
500541 if op is BRANCH :
501542 items = _fuse_branch (av )
502543 if items is not None :
503544 seq [i ] = (IN , items )
504- _fuse_difference (seq )
545+ _fuse_setops (seq , flags )
505546
506- def optimize (pattern ):
547+ def optimize (pattern , flags ):
507548 """Rewrite a parsed pattern in place and return it."""
508- _walk (pattern .data )
549+ _walk (pattern .data , flags )
509550 return pattern
0 commit comments