代码拉取完成,页面将自动刷新
package pongo2
import (
"errors"
"fmt"
"strings"
"unicode/utf8"
)
const (
TokenError = iota
EOF
TokenHTML
TokenKeyword
TokenIdentifier
TokenString
TokenNumber
TokenSymbol
TokenNil
)
var (
tokenSpaceChars = " \n\r\t"
tokenIdentifierChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"
tokenIdentifierCharsWithDigits = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789"
tokenDigits = "0123456789"
// Available symbols in pongo2 (within filters/tag)
TokenSymbols = []string{
// 3-Char symbols
"{{-", "-}}", "{%-", "-%}",
// 2-Char symbols
"==", ">=", "<=", "&&", "||", "{{", "}}", "{%", "%}", "!=", "<>",
// 1-Char symbol
"(", ")", "+", "-", "*", "<", ">", "/", "^", ",", ".", "!", "|", ":", "=", "%", "[", "]",
}
// Available keywords in pongo2
TokenKeywords = []string{"in", "and", "or", "not", "true", "false", "as", "export"}
)
type (
TokenType int
Token struct {
Filename string
Typ TokenType
Val string
Line int
Col int
TrimWhitespaces bool
}
)
type (
lexerStateFn func() lexerStateFn
lexer struct {
name string
input string
start int // start pos of the item
pos int // current pos
width int // width of last rune
tokens []*Token
errored bool
startline int
startcol int
line int
col int
inVerbatim bool
verbatimName string
}
)
func (t *Token) String() string {
val := t.Val
if len(val) > 1000 {
val = fmt.Sprintf("%s...%s", val[:10], val[len(val)-5:])
}
typ := ""
switch t.Typ {
case TokenHTML:
typ = "HTML"
case TokenError:
typ = "Error"
case TokenIdentifier:
typ = "Identifier"
case TokenKeyword:
typ = "Keyword"
case TokenNumber:
typ = "Number"
case TokenString:
typ = "String"
case TokenSymbol:
typ = "Symbol"
case TokenNil:
typ = "Nil"
default:
typ = "Unknown"
}
return fmt.Sprintf("<Token Typ=%s (%d) Val='%s' Line=%d Col=%d, WT=%t>",
typ, t.Typ, val, t.Line, t.Col, t.TrimWhitespaces)
}
func lex(name string, input string) ([]*Token, *Error) {
l := &lexer{
name: name,
input: input,
tokens: make([]*Token, 0, 100),
line: 1,
col: 1,
startline: 1,
startcol: 1,
}
l.run()
if l.errored {
errtoken := l.tokens[len(l.tokens)-1]
return nil, &Error{
Filename: name,
Line: errtoken.Line,
Column: errtoken.Col,
Sender: "lexer",
OrigError: errors.New(errtoken.Val),
}
}
return l.tokens, nil
}
func (l *lexer) value() string {
return l.input[l.start:l.pos]
}
func (l *lexer) length() int {
return l.pos - l.start
}
func (l *lexer) emit(t TokenType) {
tok := &Token{
Filename: l.name,
Typ: t,
Val: l.value(),
Line: l.startline,
Col: l.startcol,
}
if t == TokenString {
// Escape sequence \" in strings
tok.Val = strings.Replace(tok.Val, `\"`, `"`, -1)
tok.Val = strings.Replace(tok.Val, `\\`, `\`, -1)
}
if t == TokenSymbol && len(tok.Val) == 3 && (strings.HasSuffix(tok.Val, "-") || strings.HasPrefix(tok.Val, "-")) {
tok.TrimWhitespaces = true
tok.Val = strings.Replace(tok.Val, "-", "", -1)
}
l.tokens = append(l.tokens, tok)
l.start = l.pos
l.startline = l.line
l.startcol = l.col
}
func (l *lexer) next() rune {
if l.pos >= len(l.input) {
l.width = 0
return EOF
}
r, w := utf8.DecodeRuneInString(l.input[l.pos:])
l.width = w
l.pos += l.width
l.col += l.width
return r
}
func (l *lexer) backup() {
l.pos -= l.width
l.col -= l.width
}
func (l *lexer) peek() rune {
r := l.next()
l.backup()
return r
}
func (l *lexer) ignore() {
l.start = l.pos
l.startline = l.line
l.startcol = l.col
}
func (l *lexer) accept(what string) bool {
if strings.ContainsRune(what, l.next()) {
return true
}
l.backup()
return false
}
func (l *lexer) acceptRun(what string) {
for strings.ContainsRune(what, l.next()) {
}
l.backup()
}
func (l *lexer) errorf(format string, args ...any) lexerStateFn {
t := &Token{
Filename: l.name,
Typ: TokenError,
Val: fmt.Sprintf(format, args...),
Line: l.startline,
Col: l.startcol,
}
l.tokens = append(l.tokens, t)
l.errored = true
l.startline = l.line
l.startcol = l.col
return nil
}
func (l *lexer) run() {
for {
// TODO: Support verbatim tag names
// https://docs.djangoproject.com/en/dev/ref/templates/builtins/#verbatim
if l.inVerbatim {
name := l.verbatimName
if name != "" {
name += " "
}
if strings.HasPrefix(l.input[l.pos:], fmt.Sprintf("{%% endverbatim %s%%}", name)) { // end verbatim
if l.pos > l.start {
l.emit(TokenHTML)
}
w := len("{% endverbatim %}")
l.pos += w
l.col += w
l.ignore()
l.inVerbatim = false
}
} else if strings.HasPrefix(l.input[l.pos:], "{% verbatim %}") { // tag
if l.pos > l.start {
l.emit(TokenHTML)
}
l.inVerbatim = true
w := len("{% verbatim %}")
l.pos += w
l.col += w
l.ignore()
}
if !l.inVerbatim {
// Ignore single-line comments {# ... #}
if strings.HasPrefix(l.input[l.pos:], "{#") {
if l.pos > l.start {
l.emit(TokenHTML)
}
l.pos += 2 // pass '{#'
l.col += 2
for {
switch l.peek() {
case EOF:
l.errorf("Single-line comment not closed.")
return
case '\n':
l.errorf("Newline not permitted in a single-line comment.")
return
}
if strings.HasPrefix(l.input[l.pos:], "#}") {
l.pos += 2 // pass '#}'
l.col += 2
break
}
l.next()
}
l.ignore() // ignore whole comment
// Comment skipped
continue // next token
}
if strings.HasPrefix(l.input[l.pos:], "{{") || // variable
strings.HasPrefix(l.input[l.pos:], "{%") { // tag
if l.pos > l.start {
l.emit(TokenHTML)
}
l.tokenize()
if l.errored {
return
}
continue
}
}
switch l.peek() {
case '\n':
l.line++
l.col = 0
}
if l.next() == EOF {
break
}
}
if l.pos > l.start {
l.emit(TokenHTML)
}
if l.inVerbatim {
l.errorf("verbatim-tag not closed, got EOF.")
}
}
func (l *lexer) tokenize() {
for state := l.stateCode; state != nil; {
state = state()
}
}
func (l *lexer) stateCode() lexerStateFn {
outer_loop:
for {
switch {
case l.accept(tokenSpaceChars):
if l.value() == "\n" {
return l.errorf("Newline not allowed within tag/variable.")
}
l.ignore()
continue
case l.accept(tokenIdentifierChars):
return l.stateIdentifier
case l.accept(tokenDigits):
return l.stateNumber
case l.accept(`"'`):
return l.stateString
}
// Check for symbol
for _, sym := range TokenSymbols {
if strings.HasPrefix(l.input[l.start:], sym) {
l.pos += len(sym)
l.col += l.length()
l.emit(TokenSymbol)
if sym == "%}" || sym == "-%}" || sym == "}}" || sym == "-}}" {
// Tag/variable end, return after emit
return nil
}
continue outer_loop
}
}
break
}
// Normal shut down
return nil
}
func (l *lexer) stateIdentifier() lexerStateFn {
l.acceptRun(tokenIdentifierChars)
l.acceptRun(tokenIdentifierCharsWithDigits)
for _, kw := range TokenKeywords {
if kw == l.value() {
l.emit(TokenKeyword)
return l.stateCode
}
if kw == "nil" {
l.emit(TokenNil)
return l.stateCode
}
}
l.emit(TokenIdentifier)
return l.stateCode
}
func (l *lexer) stateNumber() lexerStateFn {
l.acceptRun(tokenDigits)
if l.accept(tokenIdentifierCharsWithDigits) {
// This seems to be an identifier starting with a number.
// See https://github.com/flosch/pongo2/issues/151
return l.stateIdentifier()
}
/*
Maybe context-sensitive number lexing?
* comments.0.Text // first comment
* usercomments.1.0 // second user, first comment
* if (score >= 8.5) // 8.5 as a number
if l.peek() == '.' {
l.accept(".")
if !l.accept(tokenDigits) {
return l.errorf("Malformed number.")
}
l.acceptRun(tokenDigits)
}
*/
l.emit(TokenNumber)
return l.stateCode
}
func (l *lexer) stateString() lexerStateFn {
quotationMark := l.value()
l.ignore()
l.startcol-- // we're starting the position at the first "
for !l.accept(quotationMark) {
switch l.next() {
case '\\':
// escape sequence
switch l.peek() {
case '"', '\\':
l.next()
default:
return l.errorf("Unknown escape sequence: \\%c", l.peek())
}
case EOF:
return l.errorf("Unexpected EOF, string not closed.")
case '\n':
return l.errorf("Newline in string is not allowed.")
}
}
l.backup()
l.emit(TokenString)
l.next()
l.ignore()
return l.stateCode
}
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。