2
# Secret Labs' Regular Expression Engine
4
# re-compatible interface for the sre matching engine
6
# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.
8
# This version of the SRE library can be redistributed under CNRI's
9
# Python 1.6 license. For any other use, please contact Secret Labs
10
# AB (info@pythonware.com).
12
# Portions of this engine have been developed in cooperation with
13
# CNRI. Hewlett-Packard provided funding for 1.6 integration and
14
# other compatibility work.
17
r"""Support for regular expressions (RE).
19
This module provides regular expression matching operations similar to
20
those found in Perl. It supports both 8-bit and Unicode strings; both
21
the pattern and the strings being processed can contain null bytes and
22
characters outside the US ASCII range.
24
Regular expressions can contain both special and ordinary characters.
25
Most ordinary characters, like "A", "a", or "0", are the simplest
26
regular expressions; they simply match themselves. You can
27
concatenate ordinary characters, so last matches the string 'last'.
29
The special characters are:
30
"." Matches any character except a newline.
31
"^" Matches the start of the string.
32
"$" Matches the end of the string or just before the newline at
33
the end of the string.
34
"*" Matches 0 or more (greedy) repetitions of the preceding RE.
35
Greedy means that it will match as many repetitions as possible.
36
"+" Matches 1 or more (greedy) repetitions of the preceding RE.
37
"?" Matches 0 or 1 (greedy) of the preceding RE.
38
*?,+?,?? Non-greedy versions of the previous three special characters.
39
{m,n} Matches from m to n repetitions of the preceding RE.
40
{m,n}? Non-greedy version of the above.
41
"\\" Either escapes special characters or signals a special sequence.
42
[] Indicates a set of characters.
43
A "^" as the first character indicates a complementing set.
44
"|" A|B, creates an RE that will match either A or B.
45
(...) Matches the RE inside the parentheses.
46
The contents can be retrieved or matched later in the string.
47
(?aiLmsux) The letters set the corresponding flags defined below.
48
(?:...) Non-grouping version of regular parentheses.
49
(?P<name>...) The substring matched by the group is accessible by name.
50
(?P=name) Matches the text matched earlier by the group named name.
51
(?#...) A comment; ignored.
52
(?=...) Matches if ... matches next, but doesn't consume the string.
53
(?!...) Matches if ... doesn't match next.
54
(?<=...) Matches if preceded by ... (must be fixed length).
55
(?<!...) Matches if not preceded by ... (must be fixed length).
56
(?(id/name)yes|no) Matches yes pattern if the group with id/name matched,
57
the (optional) no pattern otherwise.
59
The special sequences consist of "\\" and a character from the list
60
below. If the ordinary character is not on the list, then the
61
resulting RE will match the second character.
62
\number Matches the contents of the group of the same number.
63
\A Matches only at the start of the string.
64
\z Matches only at the end of the string.
65
\b Matches the empty string, but only at the start or end of a word.
66
\B Matches the empty string, but not at the start or end of a word.
67
\d Matches any decimal digit; equivalent to the set [0-9] in
68
bytes patterns or string patterns with the ASCII flag.
69
In string patterns without the ASCII flag, it will match the whole
70
range of Unicode digits.
71
\D Matches any non-digit character; equivalent to [^\d].
72
\s Matches any whitespace character; equivalent to [ \t\n\r\f\v] in
73
bytes patterns or string patterns with the ASCII flag.
74
In string patterns without the ASCII flag, it will match the whole
75
range of Unicode whitespace characters.
76
\S Matches any non-whitespace character; equivalent to [^\s].
77
\w Matches any alphanumeric character; equivalent to [a-zA-Z0-9_]
78
in bytes patterns or string patterns with the ASCII flag.
79
In string patterns without the ASCII flag, it will match the
80
range of Unicode alphanumeric characters (letters plus digits
82
With LOCALE, it will match the set [0-9_] plus characters defined
83
as letters for the current locale.
84
\W Matches the complement of \w.
85
\\ Matches a literal backslash.
87
This module exports the following functions:
88
prefixmatch Match a regular expression pattern to the beginning of a str.
89
match The original name of prefixmatch prior to 3.15.
90
fullmatch Match a regular expression pattern to all of a string.
91
search Search a string for the presence of a pattern.
92
sub Substitute occurrences of a pattern found in a string.
93
subn Same as sub, but also return the number of substitutions made.
94
split Split a string by the occurrences of a pattern.
95
findall Find all occurrences of a pattern in a string.
96
finditer Return an iterator yielding a Match object for each match.
97
compile Compile a pattern into a Pattern object.
98
purge Clear the regular expression cache.
99
escape Backslash all non-alphanumerics in a string.
101
Each function other than purge and escape can take an optional 'flags' argument
102
consisting of one or more of the following module constants, joined by "|".
103
A, L, and U are mutually exclusive.
104
A ASCII For string patterns, make \w, \W, \b, \B, \d, \D
105
match the corresponding ASCII character categories
106
(rather than the whole Unicode categories, which is the
108
For bytes patterns, this flag is the only available
109
behaviour and needn't be specified.
110
I IGNORECASE Perform case-insensitive matching.
111
L LOCALE Make \w, \W, \b, \B, dependent on the current locale.
112
M MULTILINE "^" matches the beginning of lines (after a newline)
113
as well as the string.
114
"$" matches the end of lines (before a newline) as well
115
as the end of the string.
116
S DOTALL "." matches any character at all, including the newline.
117
X VERBOSE Ignore whitespace and comments for nicer looking RE's.
118
U UNICODE For compatibility only. Ignored for string patterns (it
119
is the default), and forbidden for bytes patterns.
121
This module also defines exception 'PatternError', aliased to 'error' for
122
backward compatibility.
127
from . import _compiler, _parser
134
"prefixmatch", "match", "fullmatch", "search", "sub", "subn", "split",
135
"findall", "finditer", "compile", "purge", "escape",
136
"error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U",
137
"ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE",
138
"UNICODE", "NOFLAG", "RegexFlag", "PatternError"
142
@enum._simple_enum(enum.IntFlag, boundary=enum.KEEP)
145
ASCII = A = _compiler.SRE_FLAG_ASCII # assume ascii "locale"
146
IGNORECASE = I = _compiler.SRE_FLAG_IGNORECASE # ignore case
147
LOCALE = L = _compiler.SRE_FLAG_LOCALE # assume current 8-bit locale
148
UNICODE = U = _compiler.SRE_FLAG_UNICODE # assume unicode "locale"
149
MULTILINE = M = _compiler.SRE_FLAG_MULTILINE # make anchors look for newline
150
DOTALL = S = _compiler.SRE_FLAG_DOTALL # make dot match newline
151
VERBOSE = X = _compiler.SRE_FLAG_VERBOSE # ignore whitespace and comments
152
# sre extensions (experimental, don't rely on these)
153
DEBUG = _compiler.SRE_FLAG_DEBUG # dump pattern after compilation
154
__str__ = object.__str__
158
PatternError = error = _compiler.PatternError
160
# --------------------------------------------------------------------
163
def prefixmatch(pattern, string, flags=0):
164
"""Try to apply the pattern at the start of the string, returning
165
a Match object, or None if no match was found."""
166
return _compile(pattern, flags).prefixmatch(string)
168
# Our original name which was less explicitly clear about the behavior for prefixmatch.
171
def fullmatch(pattern, string, flags=0):
172
"""Try to apply the pattern to all of the string, returning
173
a Match object, or None if no match was found."""
174
return _compile(pattern, flags).fullmatch(string)
176
def search(pattern, string, flags=0):
177
"""Scan through string looking for a match to the pattern, returning
178
a Match object, or None if no match was found."""
179
1908
1908
return _compile(pattern, flags).search(string)
181
class _ZeroSentinel(int):
183
_zero_sentinel = _ZeroSentinel()
185
def sub(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel):
186
"""Return the string obtained by replacing the leftmost
187
non-overlapping occurrences of the pattern in string by the
188
replacement repl. repl can be either a string or a callable;
189
if a string, backslash escapes in it are processed. If it is
190
a callable, it's passed the Match object and must return
191
a replacement string to be used."""
193
if count is not _zero_sentinel:
194
raise TypeError("sub() got multiple values for argument 'count'")
197
if flags is not _zero_sentinel:
198
raise TypeError("sub() got multiple values for argument 'flags'")
201
raise TypeError("sub() takes from 3 to 5 positional arguments "
202
"but %d were given" % (5 + len(args)))
206
"'count' is passed as positional argument",
207
DeprecationWarning, stacklevel=2
210
return _compile(pattern, flags).sub(repl, string, count)
211
sub.__text_signature__ = '(pattern, repl, string, count=0, flags=0)'
213
def subn(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel):
214
"""Return a 2-tuple containing (new_string, number).
215
new_string is the string obtained by replacing the leftmost
216
non-overlapping occurrences of the pattern in the source
217
string by the replacement repl. number is the number of
218
substitutions that were made. repl can be either a string or a
219
callable; if a string, backslash escapes in it are processed.
220
If it is a callable, it's passed the Match object and must
221
return a replacement string to be used."""
223
if count is not _zero_sentinel:
224
raise TypeError("subn() got multiple values for argument 'count'")
227
if flags is not _zero_sentinel:
228
raise TypeError("subn() got multiple values for argument 'flags'")
231
raise TypeError("subn() takes from 3 to 5 positional arguments "
232
"but %d were given" % (5 + len(args)))
236
"'count' is passed as positional argument",
237
DeprecationWarning, stacklevel=2
240
return _compile(pattern, flags).subn(repl, string, count)
241
subn.__text_signature__ = '(pattern, repl, string, count=0, flags=0)'
243
def split(pattern, string, *args, maxsplit=_zero_sentinel, flags=_zero_sentinel):
244
"""Split the source string by the occurrences of the pattern,
245
returning a list containing the resulting substrings. If
246
capturing parentheses are used in pattern, then the text of all
247
groups in the pattern are also returned as part of the resulting
248
list. If maxsplit is nonzero, at most maxsplit splits occur,
249
and the remainder of the string is returned as the final element
252
if maxsplit is not _zero_sentinel:
253
raise TypeError("split() got multiple values for argument 'maxsplit'")
254
maxsplit, *args = args
256
if flags is not _zero_sentinel:
257
raise TypeError("split() got multiple values for argument 'flags'")
260
raise TypeError("split() takes from 2 to 4 positional arguments "
261
"but %d were given" % (4 + len(args)))
265
"'maxsplit' is passed as positional argument",
266
DeprecationWarning, stacklevel=2
269
1964
1964
return _compile(pattern, flags).split(string, maxsplit)
270
split.__text_signature__ = '(pattern, string, maxsplit=0, flags=0)'
272
def findall(pattern, string, flags=0):
273
"""Return a list of all non-overlapping matches in the string.
275
If one or more capturing groups are present in the pattern, return
276
a list of groups; this will be a list of tuples if the pattern
277
has more than one group.
279
Empty matches are included in the result."""
280
994
994
return _compile(pattern, flags).findall(string)
282
def finditer(pattern, string, flags=0):
283
"""Return an iterator over all non-overlapping matches in the
284
string. For each match, the iterator returns a Match object.
286
Empty matches are included in the result."""
287
return _compile(pattern, flags).finditer(string)
289
def compile(pattern, flags=0):
290
"Compile a regular expression pattern, returning a Pattern object."
291
return _compile(pattern, flags)
294
"Clear the regular expression caches"
297
_compile_template.cache_clear()
301
# closing ')', '}' and ']'
302
# '-' (a range in character set)
303
# '&', '~', (extended character set operations)
304
# '#' (comment) and WHITESPACE (ignored) in verbose mode
305
_special_chars_map = {i: '\\' + chr(i) for i in b'()[]{}?*+-|^$\\.&~# \t\n\r\v\f'}
309
Escape special characters in a string.
311
if isinstance(pattern, str):
312
return pattern.translate(_special_chars_map)
314
pattern = str(pattern, 'latin1')
315
return pattern.translate(_special_chars_map).encode('latin1')
317
Pattern = type(_compiler.compile('', 0))
318
Match = type(_compiler.compile('', 0).prefixmatch(''))
320
# --------------------------------------------------------------------
323
# Use the fact that dict keeps the insertion order.
324
# _cache2 uses the simple FIFO policy which has better latency.
325
# _cache uses the LRU policy which has better hit rate.
330
assert _MAXCACHE2 < _MAXCACHE
332
def _compile(pattern, flags):
333
# internal: compile pattern
334
if isinstance(flags, RegexFlag):
337
return _cache2[type(pattern), pattern, flags]
341
key = (type(pattern), pattern, flags)
342
# Item in _cache should be moved to the end if found.
343
p = _cache.pop(key, None)
345
if isinstance(pattern, Pattern):
348
"cannot process flags argument with a compiled pattern")
350
if not _compiler.isstring(pattern):
351
raise TypeError("first argument must be string or compiled pattern")
352
p = _compiler.compile(pattern, flags)
355
if len(_cache) >= _MAXCACHE:
356
# Drop the least recently used item.
357
# next(iter(_cache)) is known to have linear amortized time,
358
# but it is used here to avoid a dependency from using OrderedDict.
359
# For the small _MAXCACHE value it doesn't make much of a difference.
361
del _cache[next(iter(_cache))]
362
except (StopIteration, RuntimeError, KeyError):
367
if len(_cache2) >= _MAXCACHE2:
368
# Drop the oldest item.
370
del _cache2[next(iter(_cache2))]
371
except (StopIteration, RuntimeError, KeyError):
376
@functools.lru_cache(_MAXCACHE)
377
def _compile_template(pattern, repl):
378
# internal: compile replacement pattern
379
return _sre.template(pattern, _parser.parse_template(repl, pattern))
381
# register myself for pickling
386
return _compile, (p.pattern, p.flags)
388
copyreg.pickle(Pattern, _pickle, _compile)
390
# --------------------------------------------------------------------
391
# experimental stuff (see python-dev discussions for details)
394
def __init__(self, lexicon, flags=0):
395
from ._constants import BRANCH, SUBPATTERN
396
if isinstance(flags, RegexFlag):
398
self.lexicon = lexicon
399
# combine phrases into a compound pattern
403
for phrase, action in lexicon:
404
sub_pattern = _parser.parse(phrase, flags)
405
if sub_pattern.state.groups != 1:
406
raise ValueError("Cannot use capturing groups in re.Scanner")
408
p.append(_parser.SubPattern(s, [
409
(SUBPATTERN, (gid, 0, 0, sub_pattern)),
411
s.closegroup(gid, p[-1])
412
p = _parser.SubPattern(s, [(BRANCH, (None, p))])
413
self.scanner = _compiler.compile(p)
414
def scan(self, string):
416
append = result.append
417
_match = self.scanner.scanner(string).prefixmatch
426
action = self.lexicon[m.lastindex-1][1]
429
action = action(self, m.group())
430
if action is not None:
433
return result, string[i:]
436
def __getattr__(name):
437
if name == "__version__":
438
from warnings import _deprecated
440
_deprecated("__version__", remove=(3, 20))
441
return "2.2.1" # Do not change
442
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")