869 lines
29 KiB
Python
869 lines
29 KiB
Python
"""Implements a Jinja / Python combination lexer. The ``Lexer`` class
|
|
is used to do some preprocessing. It filters out invalid operators like
|
|
the bitshift operators we don't allow in templates. It separates
|
|
template code and python code in expressions.
|
|
"""
|
|
|
|
import re
|
|
import typing as t
|
|
from ast import literal_eval
|
|
from collections import deque
|
|
from sys import intern
|
|
|
|
from ._identifier import pattern as name_re
|
|
from .exceptions import TemplateSyntaxError
|
|
from .utils import LRUCache
|
|
|
|
if t.TYPE_CHECKING:
|
|
import typing_extensions as te
|
|
|
|
from .environment import Environment
|
|
|
|
# cache for the lexers. Exists in order to be able to have multiple
|
|
# environments with the same lexer
|
|
_lexer_cache: t.MutableMapping[t.Tuple, "Lexer"] = LRUCache(50) # type: ignore
|
|
|
|
# static regular expressions
|
|
whitespace_re = re.compile(r"\s+")
|
|
newline_re = re.compile(r"(\r\n|\r|\n)")
|
|
string_re = re.compile(
|
|
r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S
|
|
)
|
|
integer_re = re.compile(
|
|
r"""
|
|
(
|
|
0b(_?[0-1])+ # binary
|
|
|
|
|
0o(_?[0-7])+ # octal
|
|
|
|
|
0x(_?[\da-f])+ # hex
|
|
|
|
|
[1-9](_?\d)* # decimal
|
|
|
|
|
0(_?0)* # decimal zero
|
|
)
|
|
""",
|
|
re.IGNORECASE | re.VERBOSE,
|
|
)
|
|
float_re = re.compile(
|
|
r"""
|
|
(?<!\.) # doesn't start with a .
|
|
(\d+_)*\d+ # digits, possibly _ separated
|
|
(
|
|
(\.(\d+_)*\d+)? # optional fractional part
|
|
e[+\-]?(\d+_)*\d+ # exponent part
|
|
|
|
|
\.(\d+_)*\d+ # required fractional part
|
|
)
|
|
""",
|
|
re.IGNORECASE | re.VERBOSE,
|
|
)
|
|
|
|
# internal the tokens and keep references to them
|
|
TOKEN_ADD = intern("add")
|
|
TOKEN_ASSIGN = intern("assign")
|
|
TOKEN_COLON = intern("colon")
|
|
TOKEN_COMMA = intern("comma")
|
|
TOKEN_DIV = intern("div")
|
|
TOKEN_DOT = intern("dot")
|
|
TOKEN_EQ = intern("eq")
|
|
TOKEN_FLOORDIV = intern("floordiv")
|
|
TOKEN_GT = intern("gt")
|
|
TOKEN_GTEQ = intern("gteq")
|
|
TOKEN_LBRACE = intern("lbrace")
|
|
TOKEN_LBRACKET = intern("lbracket")
|
|
TOKEN_LPAREN = intern("lparen")
|
|
TOKEN_LT = intern("lt")
|
|
TOKEN_LTEQ = intern("lteq")
|
|
TOKEN_MOD = intern("mod")
|
|
TOKEN_MUL = intern("mul")
|
|
TOKEN_NE = intern("ne")
|
|
TOKEN_PIPE = intern("pipe")
|
|
TOKEN_POW = intern("pow")
|
|
TOKEN_RBRACE = intern("rbrace")
|
|
TOKEN_RBRACKET = intern("rbracket")
|
|
TOKEN_RPAREN = intern("rparen")
|
|
TOKEN_SEMICOLON = intern("semicolon")
|
|
TOKEN_SUB = intern("sub")
|
|
TOKEN_TILDE = intern("tilde")
|
|
TOKEN_WHITESPACE = intern("whitespace")
|
|
TOKEN_FLOAT = intern("float")
|
|
TOKEN_INTEGER = intern("integer")
|
|
TOKEN_NAME = intern("name")
|
|
TOKEN_STRING = intern("string")
|
|
TOKEN_OPERATOR = intern("operator")
|
|
TOKEN_BLOCK_BEGIN = intern("block_begin")
|
|
TOKEN_BLOCK_END = intern("block_end")
|
|
TOKEN_VARIABLE_BEGIN = intern("variable_begin")
|
|
TOKEN_VARIABLE_END = intern("variable_end")
|
|
TOKEN_RAW_BEGIN = intern("raw_begin")
|
|
TOKEN_RAW_END = intern("raw_end")
|
|
TOKEN_COMMENT_BEGIN = intern("comment_begin")
|
|
TOKEN_COMMENT_END = intern("comment_end")
|
|
TOKEN_COMMENT = intern("comment")
|
|
TOKEN_LINESTATEMENT_BEGIN = intern("linestatement_begin")
|
|
TOKEN_LINESTATEMENT_END = intern("linestatement_end")
|
|
TOKEN_LINECOMMENT_BEGIN = intern("linecomment_begin")
|
|
TOKEN_LINECOMMENT_END = intern("linecomment_end")
|
|
TOKEN_LINECOMMENT = intern("linecomment")
|
|
TOKEN_DATA = intern("data")
|
|
TOKEN_INITIAL = intern("initial")
|
|
TOKEN_EOF = intern("eof")
|
|
|
|
# bind operators to token types
|
|
operators = {
|
|
"+": TOKEN_ADD,
|
|
"-": TOKEN_SUB,
|
|
"/": TOKEN_DIV,
|
|
"//": TOKEN_FLOORDIV,
|
|
"*": TOKEN_MUL,
|
|
"%": TOKEN_MOD,
|
|
"**": TOKEN_POW,
|
|
"~": TOKEN_TILDE,
|
|
"[": TOKEN_LBRACKET,
|
|
"]": TOKEN_RBRACKET,
|
|
"(": TOKEN_LPAREN,
|
|
")": TOKEN_RPAREN,
|
|
"{": TOKEN_LBRACE,
|
|
"}": TOKEN_RBRACE,
|
|
"==": TOKEN_EQ,
|
|
"!=": TOKEN_NE,
|
|
">": TOKEN_GT,
|
|
">=": TOKEN_GTEQ,
|
|
"<": TOKEN_LT,
|
|
"<=": TOKEN_LTEQ,
|
|
"=": TOKEN_ASSIGN,
|
|
".": TOKEN_DOT,
|
|
":": TOKEN_COLON,
|
|
"|": TOKEN_PIPE,
|
|
",": TOKEN_COMMA,
|
|
";": TOKEN_SEMICOLON,
|
|
}
|
|
|
|
reverse_operators = {v: k for k, v in operators.items()}
|
|
assert len(operators) == len(reverse_operators), "operators dropped"
|
|
operator_re = re.compile(
|
|
f"({'|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))})"
|
|
)
|
|
|
|
ignored_tokens = frozenset(
|
|
[
|
|
TOKEN_COMMENT_BEGIN,
|
|
TOKEN_COMMENT,
|
|
TOKEN_COMMENT_END,
|
|
TOKEN_WHITESPACE,
|
|
TOKEN_LINECOMMENT_BEGIN,
|
|
TOKEN_LINECOMMENT_END,
|
|
TOKEN_LINECOMMENT,
|
|
]
|
|
)
|
|
ignore_if_empty = frozenset(
|
|
[TOKEN_WHITESPACE, TOKEN_DATA, TOKEN_COMMENT, TOKEN_LINECOMMENT]
|
|
)
|
|
|
|
|
|
def _describe_token_type(token_type: str) -> str:
|
|
if token_type in reverse_operators:
|
|
return reverse_operators[token_type]
|
|
|
|
return {
|
|
TOKEN_COMMENT_BEGIN: "begin of comment",
|
|
TOKEN_COMMENT_END: "end of comment",
|
|
TOKEN_COMMENT: "comment",
|
|
TOKEN_LINECOMMENT: "comment",
|
|
TOKEN_BLOCK_BEGIN: "begin of statement block",
|
|
TOKEN_BLOCK_END: "end of statement block",
|
|
TOKEN_VARIABLE_BEGIN: "begin of print statement",
|
|
TOKEN_VARIABLE_END: "end of print statement",
|
|
TOKEN_LINESTATEMENT_BEGIN: "begin of line statement",
|
|
TOKEN_LINESTATEMENT_END: "end of line statement",
|
|
TOKEN_DATA: "template data / text",
|
|
TOKEN_EOF: "end of template",
|
|
}.get(token_type, token_type)
|
|
|
|
|
|
def describe_token(token: "Token") -> str:
|
|
"""Returns a description of the token."""
|
|
if token.type == TOKEN_NAME:
|
|
return token.value
|
|
|
|
return _describe_token_type(token.type)
|
|
|
|
|
|
def describe_token_expr(expr: str) -> str:
|
|
"""Like `describe_token` but for token expressions."""
|
|
if ":" in expr:
|
|
type, value = expr.split(":", 1)
|
|
|
|
if type == TOKEN_NAME:
|
|
return value
|
|
else:
|
|
type = expr
|
|
|
|
return _describe_token_type(type)
|
|
|
|
|
|
def count_newlines(value: str) -> int:
|
|
"""Count the number of newline characters in the string. This is
|
|
useful for extensions that filter a stream.
|
|
"""
|
|
return len(newline_re.findall(value))
|
|
|
|
|
|
def compile_rules(environment: "Environment") -> t.List[t.Tuple[str, str]]:
|
|
"""Compiles all the rules from the environment into a list of rules."""
|
|
e = re.escape
|
|
rules = [
|
|
(
|
|
len(environment.comment_start_string),
|
|
TOKEN_COMMENT_BEGIN,
|
|
e(environment.comment_start_string),
|
|
),
|
|
(
|
|
len(environment.block_start_string),
|
|
TOKEN_BLOCK_BEGIN,
|
|
e(environment.block_start_string),
|
|
),
|
|
(
|
|
len(environment.variable_start_string),
|
|
TOKEN_VARIABLE_BEGIN,
|
|
e(environment.variable_start_string),
|
|
),
|
|
]
|
|
|
|
if environment.line_statement_prefix is not None:
|
|
rules.append(
|
|
(
|
|
len(environment.line_statement_prefix),
|
|
TOKEN_LINESTATEMENT_BEGIN,
|
|
r"^[ \t\v]*" + e(environment.line_statement_prefix),
|
|
)
|
|
)
|
|
if environment.line_comment_prefix is not None:
|
|
rules.append(
|
|
(
|
|
len(environment.line_comment_prefix),
|
|
TOKEN_LINECOMMENT_BEGIN,
|
|
r"(?:^|(?<=\S))[^\S\r\n]*" + e(environment.line_comment_prefix),
|
|
)
|
|
)
|
|
|
|
return [x[1:] for x in sorted(rules, reverse=True)]
|
|
|
|
|
|
class Failure:
|
|
"""Class that raises a `TemplateSyntaxError` if called.
|
|
Used by the `Lexer` to specify known errors.
|
|
"""
|
|
|
|
def __init__(
|
|
self, message: str, cls: t.Type[TemplateSyntaxError] = TemplateSyntaxError
|
|
) -> None:
|
|
self.message = message
|
|
self.error_class = cls
|
|
|
|
def __call__(self, lineno: int, filename: str) -> "te.NoReturn":
|
|
raise self.error_class(self.message, lineno, filename)
|
|
|
|
|
|
class Token(t.NamedTuple):
|
|
lineno: int
|
|
type: str
|
|
value: str
|
|
|
|
def __str__(self) -> str:
|
|
return describe_token(self)
|
|
|
|
def test(self, expr: str) -> bool:
|
|
"""Test a token against a token expression. This can either be a
|
|
token type or ``'token_type:token_value'``. This can only test
|
|
against string values and types.
|
|
"""
|
|
# here we do a regular string equality check as test_any is usually
|
|
# passed an iterable of not interned strings.
|
|
if self.type == expr:
|
|
return True
|
|
|
|
if ":" in expr:
|
|
return expr.split(":", 1) == [self.type, self.value]
|
|
|
|
return False
|
|
|
|
def test_any(self, *iterable: str) -> bool:
|
|
"""Test against multiple token expressions."""
|
|
return any(self.test(expr) for expr in iterable)
|
|
|
|
|
|
class TokenStreamIterator:
|
|
"""The iterator for tokenstreams. Iterate over the stream
|
|
until the eof token is reached.
|
|
"""
|
|
|
|
def __init__(self, stream: "TokenStream") -> None:
|
|
self.stream = stream
|
|
|
|
def __iter__(self) -> "TokenStreamIterator":
|
|
return self
|
|
|
|
def __next__(self) -> Token:
|
|
token = self.stream.current
|
|
|
|
if token.type is TOKEN_EOF:
|
|
self.stream.close()
|
|
raise StopIteration
|
|
|
|
next(self.stream)
|
|
return token
|
|
|
|
|
|
class TokenStream:
|
|
"""A token stream is an iterable that yields :class:`Token`\\s. The
|
|
parser however does not iterate over it but calls :meth:`next` to go
|
|
one token ahead. The current active token is stored as :attr:`current`.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
generator: t.Iterable[Token],
|
|
name: t.Optional[str],
|
|
filename: t.Optional[str],
|
|
):
|
|
self._iter = iter(generator)
|
|
self._pushed: "te.Deque[Token]" = deque()
|
|
self.name = name
|
|
self.filename = filename
|
|
self.closed = False
|
|
self.current = Token(1, TOKEN_INITIAL, "")
|
|
next(self)
|
|
|
|
def __iter__(self) -> TokenStreamIterator:
|
|
return TokenStreamIterator(self)
|
|
|
|
def __bool__(self) -> bool:
|
|
return bool(self._pushed) or self.current.type is not TOKEN_EOF
|
|
|
|
@property
|
|
def eos(self) -> bool:
|
|
"""Are we at the end of the stream?"""
|
|
return not self
|
|
|
|
def push(self, token: Token) -> None:
|
|
"""Push a token back to the stream."""
|
|
self._pushed.append(token)
|
|
|
|
def look(self) -> Token:
|
|
"""Look at the next token."""
|
|
old_token = next(self)
|
|
result = self.current
|
|
self.push(result)
|
|
self.current = old_token
|
|
return result
|
|
|
|
def skip(self, n: int = 1) -> None:
|
|
"""Got n tokens ahead."""
|
|
for _ in range(n):
|
|
next(self)
|
|
|
|
def next_if(self, expr: str) -> t.Optional[Token]:
|
|
"""Perform the token test and return the token if it matched.
|
|
Otherwise the return value is `None`.
|
|
"""
|
|
if self.current.test(expr):
|
|
return next(self)
|
|
|
|
return None
|
|
|
|
def skip_if(self, expr: str) -> bool:
|
|
"""Like :meth:`next_if` but only returns `True` or `False`."""
|
|
return self.next_if(expr) is not None
|
|
|
|
def __next__(self) -> Token:
|
|
"""Go one token ahead and return the old one.
|
|
|
|
Use the built-in :func:`next` instead of calling this directly.
|
|
"""
|
|
rv = self.current
|
|
|
|
if self._pushed:
|
|
self.current = self._pushed.popleft()
|
|
elif self.current.type is not TOKEN_EOF:
|
|
try:
|
|
self.current = next(self._iter)
|
|
except StopIteration:
|
|
self.close()
|
|
|
|
return rv
|
|
|
|
def close(self) -> None:
|
|
"""Close the stream."""
|
|
self.current = Token(self.current.lineno, TOKEN_EOF, "")
|
|
self._iter = iter(())
|
|
self.closed = True
|
|
|
|
def expect(self, expr: str) -> Token:
|
|
"""Expect a given token type and return it. This accepts the same
|
|
argument as :meth:`jinja2.lexer.Token.test`.
|
|
"""
|
|
if not self.current.test(expr):
|
|
expr = describe_token_expr(expr)
|
|
|
|
if self.current.type is TOKEN_EOF:
|
|
raise TemplateSyntaxError(
|
|
f"unexpected end of template, expected {expr!r}.",
|
|
self.current.lineno,
|
|
self.name,
|
|
self.filename,
|
|
)
|
|
|
|
raise TemplateSyntaxError(
|
|
f"expected token {expr!r}, got {describe_token(self.current)!r}",
|
|
self.current.lineno,
|
|
self.name,
|
|
self.filename,
|
|
)
|
|
|
|
return next(self)
|
|
|
|
|
|
def get_lexer(environment: "Environment") -> "Lexer":
|
|
"""Return a lexer which is probably cached."""
|
|
key = (
|
|
environment.block_start_string,
|
|
environment.block_end_string,
|
|
environment.variable_start_string,
|
|
environment.variable_end_string,
|
|
environment.comment_start_string,
|
|
environment.comment_end_string,
|
|
environment.line_statement_prefix,
|
|
environment.line_comment_prefix,
|
|
environment.trim_blocks,
|
|
environment.lstrip_blocks,
|
|
environment.newline_sequence,
|
|
environment.keep_trailing_newline,
|
|
)
|
|
lexer = _lexer_cache.get(key)
|
|
|
|
if lexer is None:
|
|
_lexer_cache[key] = lexer = Lexer(environment)
|
|
|
|
return lexer
|
|
|
|
|
|
class OptionalLStrip(tuple): # type: ignore[type-arg]
|
|
"""A special tuple for marking a point in the state that can have
|
|
lstrip applied.
|
|
"""
|
|
|
|
__slots__ = ()
|
|
|
|
# Even though it looks like a no-op, creating instances fails
|
|
# without this.
|
|
def __new__(cls, *members, **kwargs): # type: ignore
|
|
return super().__new__(cls, members)
|
|
|
|
|
|
class _Rule(t.NamedTuple):
|
|
pattern: t.Pattern[str]
|
|
tokens: t.Union[str, t.Tuple[str, ...], t.Tuple[Failure]]
|
|
command: t.Optional[str]
|
|
|
|
|
|
class Lexer:
|
|
"""Class that implements a lexer for a given environment. Automatically
|
|
created by the environment class, usually you don't have to do that.
|
|
|
|
Note that the lexer is not automatically bound to an environment.
|
|
Multiple environments can share the same lexer.
|
|
"""
|
|
|
|
def __init__(self, environment: "Environment") -> None:
|
|
# shortcuts
|
|
e = re.escape
|
|
|
|
def c(x: str) -> t.Pattern[str]:
|
|
return re.compile(x, re.M | re.S)
|
|
|
|
# lexing rules for tags
|
|
tag_rules: t.List[_Rule] = [
|
|
_Rule(whitespace_re, TOKEN_WHITESPACE, None),
|
|
_Rule(float_re, TOKEN_FLOAT, None),
|
|
_Rule(integer_re, TOKEN_INTEGER, None),
|
|
_Rule(name_re, TOKEN_NAME, None),
|
|
_Rule(string_re, TOKEN_STRING, None),
|
|
_Rule(operator_re, TOKEN_OPERATOR, None),
|
|
]
|
|
|
|
# assemble the root lexing rule. because "|" is ungreedy
|
|
# we have to sort by length so that the lexer continues working
|
|
# as expected when we have parsing rules like <% for block and
|
|
# <%= for variables. (if someone wants asp like syntax)
|
|
# variables are just part of the rules if variable processing
|
|
# is required.
|
|
root_tag_rules = compile_rules(environment)
|
|
|
|
block_start_re = e(environment.block_start_string)
|
|
block_end_re = e(environment.block_end_string)
|
|
comment_end_re = e(environment.comment_end_string)
|
|
variable_end_re = e(environment.variable_end_string)
|
|
|
|
# block suffix if trimming is enabled
|
|
block_suffix_re = "\\n?" if environment.trim_blocks else ""
|
|
|
|
self.lstrip_blocks = environment.lstrip_blocks
|
|
|
|
self.newline_sequence = environment.newline_sequence
|
|
self.keep_trailing_newline = environment.keep_trailing_newline
|
|
|
|
root_raw_re = (
|
|
rf"(?P<raw_begin>{block_start_re}(\-|\+|)\s*raw\s*"
|
|
rf"(?:\-{block_end_re}\s*|{block_end_re}))"
|
|
)
|
|
root_parts_re = "|".join(
|
|
[root_raw_re] + [rf"(?P<{n}>{r}(\-|\+|))" for n, r in root_tag_rules]
|
|
)
|
|
|
|
# global lexing rules
|
|
self.rules: t.Dict[str, t.List[_Rule]] = {
|
|
"root": [
|
|
# directives
|
|
_Rule(
|
|
c(rf"(.*?)(?:{root_parts_re})"),
|
|
OptionalLStrip(TOKEN_DATA, "#bygroup"), # type: ignore
|
|
"#bygroup",
|
|
),
|
|
# data
|
|
_Rule(c(".+"), TOKEN_DATA, None),
|
|
],
|
|
# comments
|
|
TOKEN_COMMENT_BEGIN: [
|
|
_Rule(
|
|
c(
|
|
rf"(.*?)((?:\+{comment_end_re}|\-{comment_end_re}\s*"
|
|
rf"|{comment_end_re}{block_suffix_re}))"
|
|
),
|
|
(TOKEN_COMMENT, TOKEN_COMMENT_END),
|
|
"#pop",
|
|
),
|
|
_Rule(c(r"(.)"), (Failure("Missing end of comment tag"),), None),
|
|
],
|
|
# blocks
|
|
TOKEN_BLOCK_BEGIN: [
|
|
_Rule(
|
|
c(
|
|
rf"(?:\+{block_end_re}|\-{block_end_re}\s*"
|
|
rf"|{block_end_re}{block_suffix_re})"
|
|
),
|
|
TOKEN_BLOCK_END,
|
|
"#pop",
|
|
),
|
|
]
|
|
+ tag_rules,
|
|
# variables
|
|
TOKEN_VARIABLE_BEGIN: [
|
|
_Rule(
|
|
c(rf"\-{variable_end_re}\s*|{variable_end_re}"),
|
|
TOKEN_VARIABLE_END,
|
|
"#pop",
|
|
)
|
|
]
|
|
+ tag_rules,
|
|
# raw block
|
|
TOKEN_RAW_BEGIN: [
|
|
_Rule(
|
|
c(
|
|
rf"(.*?)((?:{block_start_re}(\-|\+|))\s*endraw\s*"
|
|
rf"(?:\+{block_end_re}|\-{block_end_re}\s*"
|
|
rf"|{block_end_re}{block_suffix_re}))"
|
|
),
|
|
OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END), # type: ignore
|
|
"#pop",
|
|
),
|
|
_Rule(c(r"(.)"), (Failure("Missing end of raw directive"),), None),
|
|
],
|
|
# line statements
|
|
TOKEN_LINESTATEMENT_BEGIN: [
|
|
_Rule(c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop")
|
|
]
|
|
+ tag_rules,
|
|
# line comments
|
|
TOKEN_LINECOMMENT_BEGIN: [
|
|
_Rule(
|
|
c(r"(.*?)()(?=\n|$)"),
|
|
(TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END),
|
|
"#pop",
|
|
)
|
|
],
|
|
}
|
|
|
|
def _normalize_newlines(self, value: str) -> str:
|
|
"""Replace all newlines with the configured sequence in strings
|
|
and template data.
|
|
"""
|
|
return newline_re.sub(self.newline_sequence, value)
|
|
|
|
def tokenize(
|
|
self,
|
|
source: str,
|
|
name: t.Optional[str] = None,
|
|
filename: t.Optional[str] = None,
|
|
state: t.Optional[str] = None,
|
|
) -> TokenStream:
|
|
"""Calls tokeniter + tokenize and wraps it in a token stream."""
|
|
stream = self.tokeniter(source, name, filename, state)
|
|
return TokenStream(self.wrap(stream, name, filename), name, filename)
|
|
|
|
def wrap(
|
|
self,
|
|
stream: t.Iterable[t.Tuple[int, str, str]],
|
|
name: t.Optional[str] = None,
|
|
filename: t.Optional[str] = None,
|
|
) -> t.Iterator[Token]:
|
|
"""This is called with the stream as returned by `tokenize` and wraps
|
|
every token in a :class:`Token` and converts the value.
|
|
"""
|
|
for lineno, token, value_str in stream:
|
|
if token in ignored_tokens:
|
|
continue
|
|
|
|
value: t.Any = value_str
|
|
|
|
if token == TOKEN_LINESTATEMENT_BEGIN:
|
|
token = TOKEN_BLOCK_BEGIN
|
|
elif token == TOKEN_LINESTATEMENT_END:
|
|
token = TOKEN_BLOCK_END
|
|
# we are not interested in those tokens in the parser
|
|
elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END):
|
|
continue
|
|
elif token == TOKEN_DATA:
|
|
value = self._normalize_newlines(value_str)
|
|
elif token == "keyword":
|
|
token = value_str
|
|
elif token == TOKEN_NAME:
|
|
value = value_str
|
|
|
|
if not value.isidentifier():
|
|
raise TemplateSyntaxError(
|
|
"Invalid character in identifier", lineno, name, filename
|
|
)
|
|
elif token == TOKEN_STRING:
|
|
# try to unescape string
|
|
try:
|
|
value = (
|
|
self._normalize_newlines(value_str[1:-1])
|
|
.encode("ascii", "backslashreplace")
|
|
.decode("unicode-escape")
|
|
)
|
|
except Exception as e:
|
|
msg = str(e).split(":")[-1].strip()
|
|
raise TemplateSyntaxError(msg, lineno, name, filename) from e
|
|
elif token == TOKEN_INTEGER:
|
|
value = int(value_str.replace("_", ""), 0)
|
|
elif token == TOKEN_FLOAT:
|
|
# remove all "_" first to support more Python versions
|
|
value = literal_eval(value_str.replace("_", ""))
|
|
elif token == TOKEN_OPERATOR:
|
|
token = operators[value_str]
|
|
|
|
yield Token(lineno, token, value)
|
|
|
|
def tokeniter(
|
|
self,
|
|
source: str,
|
|
name: t.Optional[str],
|
|
filename: t.Optional[str] = None,
|
|
state: t.Optional[str] = None,
|
|
) -> t.Iterator[t.Tuple[int, str, str]]:
|
|
"""This method tokenizes the text and returns the tokens in a
|
|
generator. Use this method if you just want to tokenize a template.
|
|
|
|
.. versionchanged:: 3.0
|
|
Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line
|
|
breaks.
|
|
"""
|
|
lines = newline_re.split(source)[::2]
|
|
|
|
if not self.keep_trailing_newline and lines[-1] == "":
|
|
del lines[-1]
|
|
|
|
source = "\n".join(lines)
|
|
pos = 0
|
|
lineno = 1
|
|
stack = ["root"]
|
|
|
|
if state is not None and state != "root":
|
|
assert state in ("variable", "block"), "invalid state"
|
|
stack.append(state + "_begin")
|
|
|
|
statetokens = self.rules[stack[-1]]
|
|
source_length = len(source)
|
|
balancing_stack: t.List[str] = []
|
|
newlines_stripped = 0
|
|
line_starting = True
|
|
|
|
while True:
|
|
# tokenizer loop
|
|
for regex, tokens, new_state in statetokens:
|
|
m = regex.match(source, pos)
|
|
|
|
# if no match we try again with the next rule
|
|
if m is None:
|
|
continue
|
|
|
|
# we only match blocks and variables if braces / parentheses
|
|
# are balanced. continue parsing with the lower rule which
|
|
# is the operator rule. do this only if the end tags look
|
|
# like operators
|
|
if balancing_stack and tokens in (
|
|
TOKEN_VARIABLE_END,
|
|
TOKEN_BLOCK_END,
|
|
TOKEN_LINESTATEMENT_END,
|
|
):
|
|
continue
|
|
|
|
# tuples support more options
|
|
if isinstance(tokens, tuple):
|
|
groups: t.Sequence[str] = m.groups()
|
|
|
|
if isinstance(tokens, OptionalLStrip):
|
|
# Rule supports lstrip. Match will look like
|
|
# text, block type, whitespace control, type, control, ...
|
|
text = groups[0]
|
|
# Skipping the text and first type, every other group is the
|
|
# whitespace control for each type. One of the groups will be
|
|
# -, +, or empty string instead of None.
|
|
strip_sign = next(g for g in groups[2::2] if g is not None)
|
|
|
|
if strip_sign == "-":
|
|
# Strip all whitespace between the text and the tag.
|
|
stripped = text.rstrip()
|
|
newlines_stripped = text[len(stripped) :].count("\n")
|
|
groups = [stripped, *groups[1:]]
|
|
elif (
|
|
# Not marked for preserving whitespace.
|
|
strip_sign != "+"
|
|
# lstrip is enabled.
|
|
and self.lstrip_blocks
|
|
# Not a variable expression.
|
|
and not m.groupdict().get(TOKEN_VARIABLE_BEGIN)
|
|
):
|
|
# The start of text between the last newline and the tag.
|
|
l_pos = text.rfind("\n") + 1
|
|
|
|
if l_pos > 0 or line_starting:
|
|
# If there's only whitespace between the newline and the
|
|
# tag, strip it.
|
|
if whitespace_re.fullmatch(text, l_pos):
|
|
groups = [text[:l_pos], *groups[1:]]
|
|
|
|
for idx, token in enumerate(tokens):
|
|
# failure group
|
|
if token.__class__ is Failure:
|
|
raise token(lineno, filename)
|
|
# bygroup is a bit more complex, in that case we
|
|
# yield for the current token the first named
|
|
# group that matched
|
|
elif token == "#bygroup":
|
|
for key, value in m.groupdict().items():
|
|
if value is not None:
|
|
yield lineno, key, value
|
|
lineno += value.count("\n")
|
|
break
|
|
else:
|
|
raise RuntimeError(
|
|
f"{regex!r} wanted to resolve the token dynamically"
|
|
" but no group matched"
|
|
)
|
|
# normal group
|
|
else:
|
|
data = groups[idx]
|
|
|
|
if data or token not in ignore_if_empty:
|
|
yield lineno, token, data
|
|
|
|
lineno += data.count("\n") + newlines_stripped
|
|
newlines_stripped = 0
|
|
|
|
# strings as token just are yielded as it.
|
|
else:
|
|
data = m.group()
|
|
|
|
# update brace/parentheses balance
|
|
if tokens == TOKEN_OPERATOR:
|
|
if data == "{":
|
|
balancing_stack.append("}")
|
|
elif data == "(":
|
|
balancing_stack.append(")")
|
|
elif data == "[":
|
|
balancing_stack.append("]")
|
|
elif data in ("}", ")", "]"):
|
|
if not balancing_stack:
|
|
raise TemplateSyntaxError(
|
|
f"unexpected '{data}'", lineno, name, filename
|
|
)
|
|
|
|
expected_op = balancing_stack.pop()
|
|
|
|
if expected_op != data:
|
|
raise TemplateSyntaxError(
|
|
f"unexpected '{data}', expected '{expected_op}'",
|
|
lineno,
|
|
name,
|
|
filename,
|
|
)
|
|
|
|
# yield items
|
|
if data or tokens not in ignore_if_empty:
|
|
yield lineno, tokens, data
|
|
|
|
lineno += data.count("\n")
|
|
|
|
line_starting = m.group()[-1:] == "\n"
|
|
# fetch new position into new variable so that we can check
|
|
# if there is a internal parsing error which would result
|
|
# in an infinite loop
|
|
pos2 = m.end()
|
|
|
|
# handle state changes
|
|
if new_state is not None:
|
|
# remove the uppermost state
|
|
if new_state == "#pop":
|
|
stack.pop()
|
|
# resolve the new state by group checking
|
|
elif new_state == "#bygroup":
|
|
for key, value in m.groupdict().items():
|
|
if value is not None:
|
|
stack.append(key)
|
|
break
|
|
else:
|
|
raise RuntimeError(
|
|
f"{regex!r} wanted to resolve the new state dynamically"
|
|
f" but no group matched"
|
|
)
|
|
# direct state name given
|
|
else:
|
|
stack.append(new_state)
|
|
|
|
statetokens = self.rules[stack[-1]]
|
|
# we are still at the same position and no stack change.
|
|
# this means a loop without break condition, avoid that and
|
|
# raise error
|
|
elif pos2 == pos:
|
|
raise RuntimeError(
|
|
f"{regex!r} yielded empty string without stack change"
|
|
)
|
|
|
|
# publish new function and start again
|
|
pos = pos2
|
|
break
|
|
# if loop terminated without break we haven't found a single match
|
|
# either we are at the end of the file or we have a problem
|
|
else:
|
|
# end of text
|
|
if pos >= source_length:
|
|
return
|
|
|
|
# something went wrong
|
|
raise TemplateSyntaxError(
|
|
f"unexpected char {source[pos]!r} at {pos}", lineno, name, filename
|
|
)
|