[3] | 1 | # $Id: Parser.py,v 1.137 2008/03/10 05:25:13 tavis_rudd Exp $ |
---|
| 2 | """Parser classes for Cheetah's Compiler |
---|
| 3 | |
---|
| 4 | Classes: |
---|
| 5 | ParseError( Exception ) |
---|
| 6 | _LowLevelParser( Cheetah.SourceReader.SourceReader ), basically a lexer |
---|
| 7 | _HighLevelParser( _LowLevelParser ) |
---|
| 8 | Parser === _HighLevelParser (an alias) |
---|
| 9 | |
---|
| 10 | Meta-Data |
---|
| 11 | ================================================================================ |
---|
| 12 | Author: Tavis Rudd <tavis@damnsimple.com> |
---|
| 13 | Version: $Revision: 1.137 $ |
---|
| 14 | Start Date: 2001/08/01 |
---|
| 15 | Last Revision Date: $Date: 2008/03/10 05:25:13 $ |
---|
| 16 | """ |
---|
| 17 | __author__ = "Tavis Rudd <tavis@damnsimple.com>" |
---|
| 18 | __revision__ = "$Revision: 1.137 $"[11:-2] |
---|
| 19 | |
---|
| 20 | import os |
---|
| 21 | import sys |
---|
| 22 | import re |
---|
| 23 | from re import DOTALL, MULTILINE |
---|
| 24 | from types import StringType, ListType, TupleType, ClassType, TypeType |
---|
| 25 | import time |
---|
| 26 | from tokenize import pseudoprog |
---|
| 27 | import inspect |
---|
| 28 | import new |
---|
| 29 | import traceback |
---|
| 30 | |
---|
| 31 | from Cheetah.SourceReader import SourceReader |
---|
| 32 | from Cheetah import Filters |
---|
| 33 | from Cheetah import ErrorCatchers |
---|
| 34 | from Cheetah.Unspecified import Unspecified |
---|
| 35 | from Cheetah.Macros.I18n import I18n |
---|
| 36 | |
---|
| 37 | # re tools |
---|
| 38 | _regexCache = {} |
---|
| 39 | def cachedRegex(pattern): |
---|
| 40 | if pattern not in _regexCache: |
---|
| 41 | _regexCache[pattern] = re.compile(pattern) |
---|
| 42 | return _regexCache[pattern] |
---|
| 43 | |
---|
| 44 | def escapeRegexChars(txt, |
---|
| 45 | escapeRE=re.compile(r'([\$\^\*\+\.\?\{\}\[\]\(\)\|\\])')): |
---|
| 46 | |
---|
| 47 | """Return a txt with all special regular expressions chars escaped.""" |
---|
| 48 | |
---|
| 49 | return escapeRE.sub(r'\\\1' , txt) |
---|
| 50 | |
---|
| 51 | def group(*choices): return '(' + '|'.join(choices) + ')' |
---|
| 52 | def nongroup(*choices): return '(?:' + '|'.join(choices) + ')' |
---|
| 53 | def namedGroup(name, *choices): return '(P:<' + name +'>' + '|'.join(choices) + ')' |
---|
| 54 | def any(*choices): return apply(group, choices) + '*' |
---|
| 55 | def maybe(*choices): return apply(group, choices) + '?' |
---|
| 56 | |
---|
| 57 | ################################################## |
---|
| 58 | ## CONSTANTS & GLOBALS ## |
---|
| 59 | |
---|
| 60 | NO_CACHE = 0 |
---|
| 61 | STATIC_CACHE = 1 |
---|
| 62 | REFRESH_CACHE = 2 |
---|
| 63 | |
---|
| 64 | SET_LOCAL = 0 |
---|
| 65 | SET_GLOBAL = 1 |
---|
| 66 | SET_MODULE = 2 |
---|
| 67 | |
---|
| 68 | ################################################## |
---|
| 69 | ## Tokens for the parser ## |
---|
| 70 | |
---|
| 71 | #generic |
---|
| 72 | identchars = "abcdefghijklmnopqrstuvwxyz" \ |
---|
| 73 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ_" |
---|
| 74 | namechars = identchars + "0123456789" |
---|
| 75 | |
---|
| 76 | #operators |
---|
| 77 | powerOp = '**' |
---|
| 78 | unaryArithOps = ('+', '-', '~') |
---|
| 79 | binaryArithOps = ('+', '-', '/', '//','%') |
---|
| 80 | shiftOps = ('>>','<<') |
---|
| 81 | bitwiseOps = ('&','|','^') |
---|
| 82 | assignOp = '=' |
---|
| 83 | augAssignOps = ('+=','-=','/=','*=', '**=','^=','%=', |
---|
| 84 | '>>=','<<=','&=','|=', ) |
---|
| 85 | assignmentOps = (assignOp,) + augAssignOps |
---|
| 86 | |
---|
| 87 | compOps = ('<','>','==','!=','<=','>=', '<>', 'is', 'in',) |
---|
| 88 | booleanOps = ('and','or','not') |
---|
| 89 | operators = (powerOp,) + unaryArithOps + binaryArithOps \ |
---|
| 90 | + shiftOps + bitwiseOps + assignmentOps \ |
---|
| 91 | + compOps + booleanOps |
---|
| 92 | |
---|
| 93 | delimeters = ('(',')','{','}','[',']', |
---|
| 94 | ',','.',':',';','=','`') + augAssignOps |
---|
| 95 | |
---|
| 96 | |
---|
| 97 | keywords = ('and', 'del', 'for', 'is', 'raise', |
---|
| 98 | 'assert', 'elif', 'from', 'lambda', 'return', |
---|
| 99 | 'break', 'else', 'global', 'not', 'try', |
---|
| 100 | 'class', 'except', 'if', 'or', 'while', |
---|
| 101 | 'continue', 'exec', 'import', 'pass', |
---|
| 102 | 'def', 'finally', 'in', 'print', |
---|
| 103 | ) |
---|
| 104 | |
---|
| 105 | single3 = "'''" |
---|
| 106 | double3 = '"""' |
---|
| 107 | |
---|
| 108 | tripleQuotedStringStarts = ("'''", '"""', |
---|
| 109 | "r'''", 'r"""', "R'''", 'R"""', |
---|
| 110 | "u'''", 'u"""', "U'''", 'U"""', |
---|
| 111 | "ur'''", 'ur"""', "Ur'''", 'Ur"""', |
---|
| 112 | "uR'''", 'uR"""', "UR'''", 'UR"""') |
---|
| 113 | |
---|
| 114 | tripleQuotedStringPairs = {"'''": single3, '"""': double3, |
---|
| 115 | "r'''": single3, 'r"""': double3, |
---|
| 116 | "u'''": single3, 'u"""': double3, |
---|
| 117 | "ur'''": single3, 'ur"""': double3, |
---|
| 118 | "R'''": single3, 'R"""': double3, |
---|
| 119 | "U'''": single3, 'U"""': double3, |
---|
| 120 | "uR'''": single3, 'uR"""': double3, |
---|
| 121 | "Ur'''": single3, 'Ur"""': double3, |
---|
| 122 | "UR'''": single3, 'UR"""': double3, |
---|
| 123 | } |
---|
| 124 | |
---|
| 125 | closurePairs= {')':'(',']':'[','}':'{'} |
---|
| 126 | closurePairsRev= {'(':')','[':']','{':'}'} |
---|
| 127 | |
---|
| 128 | ################################################## |
---|
| 129 | ## Regex chunks for the parser ## |
---|
| 130 | |
---|
| 131 | tripleQuotedStringREs = {} |
---|
| 132 | def makeTripleQuoteRe(start, end): |
---|
| 133 | start = escapeRegexChars(start) |
---|
| 134 | end = escapeRegexChars(end) |
---|
| 135 | return re.compile(r'(?:' + start + r').*?' + r'(?:' + end + r')', re.DOTALL) |
---|
| 136 | |
---|
| 137 | for start, end in tripleQuotedStringPairs.items(): |
---|
| 138 | tripleQuotedStringREs[start] = makeTripleQuoteRe(start, end) |
---|
| 139 | |
---|
| 140 | WS = r'[ \f\t]*' |
---|
| 141 | EOL = r'\r\n|\n|\r' |
---|
| 142 | EOLZ = EOL + r'|\Z' |
---|
| 143 | escCharLookBehind = nongroup(r'(?<=\A)',r'(?<!\\)') |
---|
| 144 | nameCharLookAhead = r'(?=[A-Za-z_])' |
---|
| 145 | identRE=re.compile(r'[a-zA-Z_][a-zA-Z_0-9]*') |
---|
| 146 | EOLre=re.compile(r'(?:\r\n|\r|\n)') |
---|
| 147 | |
---|
| 148 | specialVarRE=re.compile(r'([a-zA-z_]+)@') # for matching specialVar comments |
---|
| 149 | # e.g. ##author@ Tavis Rudd |
---|
| 150 | |
---|
| 151 | unicodeDirectiveRE = re.compile( |
---|
| 152 | r'(?:^|\r\n|\r|\n)\s*#\s{0,5}unicode[:\s]*([-\w.]*)\s*(?:\r\n|\r|\n)', re.MULTILINE) |
---|
| 153 | encodingDirectiveRE = re.compile( |
---|
| 154 | r'(?:^|\r\n|\r|\n)\s*#\s{0,5}encoding[:\s]*([-\w.]*)\s*(?:\r\n|\r|\n)', re.MULTILINE) |
---|
| 155 | |
---|
| 156 | escapedNewlineRE = re.compile(r'(?<!\\)\\n') |
---|
| 157 | |
---|
| 158 | directiveNamesAndParsers = { |
---|
| 159 | # importing and inheritance |
---|
| 160 | 'import':None, |
---|
| 161 | 'from':None, |
---|
| 162 | 'extends': 'eatExtends', |
---|
| 163 | 'implements': 'eatImplements', |
---|
| 164 | 'super': 'eatSuper', |
---|
| 165 | |
---|
| 166 | # output, filtering, and caching |
---|
| 167 | 'slurp': 'eatSlurp', |
---|
| 168 | 'raw': 'eatRaw', |
---|
| 169 | 'include': 'eatInclude', |
---|
| 170 | 'cache': 'eatCache', |
---|
| 171 | 'filter': 'eatFilter', |
---|
| 172 | 'echo': None, |
---|
| 173 | 'silent': None, |
---|
| 174 | 'transform' : 'eatTransform', |
---|
| 175 | |
---|
| 176 | 'call': 'eatCall', |
---|
| 177 | 'arg': 'eatCallArg', |
---|
| 178 | |
---|
| 179 | 'capture': 'eatCapture', |
---|
| 180 | |
---|
| 181 | # declaration, assignment, and deletion |
---|
| 182 | 'attr': 'eatAttr', |
---|
| 183 | 'def': 'eatDef', |
---|
| 184 | 'block': 'eatBlock', |
---|
| 185 | '@': 'eatDecorator', |
---|
| 186 | 'defmacro': 'eatDefMacro', |
---|
| 187 | |
---|
| 188 | 'closure': 'eatClosure', |
---|
| 189 | |
---|
| 190 | 'set': 'eatSet', |
---|
| 191 | 'del': None, |
---|
| 192 | |
---|
| 193 | # flow control |
---|
| 194 | 'if': 'eatIf', |
---|
| 195 | 'while': None, |
---|
| 196 | 'for': None, |
---|
| 197 | 'else': None, |
---|
| 198 | 'elif': None, |
---|
| 199 | 'pass': None, |
---|
| 200 | 'break': None, |
---|
| 201 | 'continue': None, |
---|
| 202 | 'stop': None, |
---|
| 203 | 'return': None, |
---|
| 204 | 'yield': None, |
---|
| 205 | |
---|
| 206 | # little wrappers |
---|
| 207 | 'repeat': None, |
---|
| 208 | 'unless': None, |
---|
| 209 | |
---|
| 210 | # error handling |
---|
| 211 | 'assert': None, |
---|
| 212 | 'raise': None, |
---|
| 213 | 'try': None, |
---|
| 214 | 'except': None, |
---|
| 215 | 'finally': None, |
---|
| 216 | 'errorCatcher': 'eatErrorCatcher', |
---|
| 217 | |
---|
| 218 | # intructions to the parser and compiler |
---|
| 219 | 'breakpoint': 'eatBreakPoint', |
---|
| 220 | 'compiler': 'eatCompiler', |
---|
| 221 | 'compiler-settings': 'eatCompilerSettings', |
---|
| 222 | |
---|
| 223 | # misc |
---|
| 224 | 'shBang': 'eatShbang', |
---|
| 225 | 'encoding': 'eatEncoding', |
---|
| 226 | |
---|
| 227 | 'end': 'eatEndDirective', |
---|
| 228 | } |
---|
| 229 | |
---|
| 230 | endDirectiveNamesAndHandlers = { |
---|
| 231 | 'def': 'handleEndDef', # has short-form |
---|
| 232 | 'block': None, # has short-form |
---|
| 233 | 'closure': None, # has short-form |
---|
| 234 | 'cache': None, # has short-form |
---|
| 235 | 'call': None, # has short-form |
---|
| 236 | 'capture': None, # has short-form |
---|
| 237 | 'filter': None, |
---|
| 238 | 'errorCatcher':None, |
---|
| 239 | 'while': None, # has short-form |
---|
| 240 | 'for': None, # has short-form |
---|
| 241 | 'if': None, # has short-form |
---|
| 242 | 'try': None, # has short-form |
---|
| 243 | 'repeat': None, # has short-form |
---|
| 244 | 'unless': None, # has short-form |
---|
| 245 | } |
---|
| 246 | |
---|
| 247 | ################################################## |
---|
| 248 | ## CLASSES ## |
---|
| 249 | |
---|
| 250 | # @@TR: SyntaxError doesn't call exception.__str__ for some reason! |
---|
| 251 | #class ParseError(SyntaxError): |
---|
| 252 | class ParseError(ValueError): |
---|
| 253 | def __init__(self, stream, msg='Invalid Syntax', extMsg='', lineno=None, col=None): |
---|
| 254 | self.stream = stream |
---|
| 255 | if stream.pos() >= len(stream): |
---|
| 256 | stream.setPos(len(stream) -1) |
---|
| 257 | self.msg = msg |
---|
| 258 | self.extMsg = extMsg |
---|
| 259 | self.lineno = lineno |
---|
| 260 | self.col = col |
---|
| 261 | |
---|
| 262 | def __str__(self): |
---|
| 263 | return self.report() |
---|
| 264 | |
---|
| 265 | def report(self): |
---|
| 266 | stream = self.stream |
---|
| 267 | if stream.filename(): |
---|
| 268 | f = " in file %s" % stream.filename() |
---|
| 269 | else: |
---|
| 270 | f = '' |
---|
| 271 | report = '' |
---|
| 272 | if self.lineno: |
---|
| 273 | lineno = self.lineno |
---|
| 274 | row, col, line = (lineno, (self.col or 0), |
---|
| 275 | self.stream.splitlines()[lineno-1]) |
---|
| 276 | else: |
---|
| 277 | row, col, line = self.stream.getRowColLine() |
---|
| 278 | |
---|
| 279 | ## get the surrounding lines |
---|
| 280 | lines = stream.splitlines() |
---|
| 281 | prevLines = [] # (rowNum, content) |
---|
| 282 | for i in range(1,4): |
---|
| 283 | if row-1-i <=0: |
---|
| 284 | break |
---|
| 285 | prevLines.append( (row-i,lines[row-1-i]) ) |
---|
| 286 | |
---|
| 287 | nextLines = [] # (rowNum, content) |
---|
| 288 | for i in range(1,4): |
---|
| 289 | if not row-1+i < len(lines): |
---|
| 290 | break |
---|
| 291 | nextLines.append( (row+i,lines[row-1+i]) ) |
---|
| 292 | nextLines.reverse() |
---|
| 293 | |
---|
| 294 | ## print the main message |
---|
| 295 | report += "\n\n%s\n" %self.msg |
---|
| 296 | report += "Line %i, column %i%s\n\n" % (row, col, f) |
---|
| 297 | report += 'Line|Cheetah Code\n' |
---|
| 298 | report += '----|-------------------------------------------------------------\n' |
---|
| 299 | while prevLines: |
---|
| 300 | lineInfo = prevLines.pop() |
---|
| 301 | report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]} |
---|
| 302 | report += "%(row)-4d|%(line)s\n"% {'row':row, 'line':line} |
---|
| 303 | report += ' '*5 +' '*(col-1) + "^\n" |
---|
| 304 | |
---|
| 305 | while nextLines: |
---|
| 306 | lineInfo = nextLines.pop() |
---|
| 307 | report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]} |
---|
| 308 | ## add the extra msg |
---|
| 309 | if self.extMsg: |
---|
| 310 | report += self.extMsg + '\n' |
---|
| 311 | |
---|
| 312 | return report |
---|
| 313 | |
---|
| 314 | class ForbiddenSyntax(ParseError): pass |
---|
| 315 | class ForbiddenExpression(ForbiddenSyntax): pass |
---|
| 316 | class ForbiddenDirective(ForbiddenSyntax): pass |
---|
| 317 | |
---|
| 318 | class CheetahVariable: |
---|
| 319 | def __init__(self, nameChunks, useNameMapper=True, cacheToken=None, |
---|
| 320 | rawSource=None): |
---|
| 321 | self.nameChunks = nameChunks |
---|
| 322 | self.useNameMapper = useNameMapper |
---|
| 323 | self.cacheToken = cacheToken |
---|
| 324 | self.rawSource = rawSource |
---|
| 325 | |
---|
| 326 | class Placeholder(CheetahVariable): pass |
---|
| 327 | |
---|
| 328 | class ArgList: |
---|
| 329 | """Used by _LowLevelParser.getArgList()""" |
---|
| 330 | |
---|
| 331 | def __init__(self): |
---|
| 332 | self.argNames = [] |
---|
| 333 | self.defVals = [] |
---|
| 334 | self.i = 0 |
---|
| 335 | |
---|
| 336 | def addArgName(self, name): |
---|
| 337 | self.argNames.append( name ) |
---|
| 338 | self.defVals.append( None ) |
---|
| 339 | |
---|
| 340 | def next(self): |
---|
| 341 | self.i += 1 |
---|
| 342 | |
---|
| 343 | def addToDefVal(self, token): |
---|
| 344 | i = self.i |
---|
| 345 | if self.defVals[i] == None: |
---|
| 346 | self.defVals[i] = '' |
---|
| 347 | self.defVals[i] += token |
---|
| 348 | |
---|
| 349 | def merge(self): |
---|
| 350 | defVals = self.defVals |
---|
| 351 | for i in range(len(defVals)): |
---|
| 352 | if type(defVals[i]) == StringType: |
---|
| 353 | defVals[i] = defVals[i].strip() |
---|
| 354 | |
---|
| 355 | return map(None, [i.strip() for i in self.argNames], defVals) |
---|
| 356 | |
---|
| 357 | def __str__(self): |
---|
| 358 | return str(self.merge()) |
---|
| 359 | |
---|
| 360 | class _LowLevelParser(SourceReader): |
---|
| 361 | """This class implements the methods to match or extract ('get*') the basic |
---|
| 362 | elements of Cheetah's grammar. It does NOT handle any code generation or |
---|
| 363 | state management. |
---|
| 364 | """ |
---|
| 365 | |
---|
| 366 | _settingsManager = None |
---|
| 367 | |
---|
| 368 | def setSettingsManager(self, settingsManager): |
---|
| 369 | self._settingsManager = settingsManager |
---|
| 370 | |
---|
| 371 | def setting(self, key, default=Unspecified): |
---|
| 372 | if default is Unspecified: |
---|
| 373 | return self._settingsManager.setting(key) |
---|
| 374 | else: |
---|
| 375 | return self._settingsManager.setting(key, default=default) |
---|
| 376 | |
---|
| 377 | def setSetting(self, key, val): |
---|
| 378 | self._settingsManager.setSetting(key, val) |
---|
| 379 | |
---|
| 380 | def settings(self): |
---|
| 381 | return self._settingsManager.settings() |
---|
| 382 | |
---|
| 383 | def updateSettings(self, settings): |
---|
| 384 | self._settingsManager.updateSettings(settings) |
---|
| 385 | |
---|
| 386 | def _initializeSettings(self): |
---|
| 387 | self._settingsManager._initializeSettings() |
---|
| 388 | |
---|
| 389 | def configureParser(self): |
---|
| 390 | """Is called by the Compiler instance after the parser has had a |
---|
| 391 | settingsManager assigned with self.setSettingsManager() |
---|
| 392 | """ |
---|
| 393 | self._makeCheetahVarREs() |
---|
| 394 | self._makeCommentREs() |
---|
| 395 | self._makeDirectiveREs() |
---|
| 396 | self._makePspREs() |
---|
| 397 | self._possibleNonStrConstantChars = ( |
---|
| 398 | self.setting('commentStartToken')[0] + |
---|
| 399 | self.setting('multiLineCommentStartToken')[0] + |
---|
| 400 | self.setting('cheetahVarStartToken')[0] + |
---|
| 401 | self.setting('directiveStartToken')[0] + |
---|
| 402 | self.setting('PSPStartToken')[0]) |
---|
| 403 | self._nonStrConstMatchers = [ |
---|
| 404 | self.matchCommentStartToken, |
---|
| 405 | self.matchMultiLineCommentStartToken, |
---|
| 406 | self.matchVariablePlaceholderStart, |
---|
| 407 | self.matchExpressionPlaceholderStart, |
---|
| 408 | self.matchDirective, |
---|
| 409 | self.matchPSPStartToken, |
---|
| 410 | self.matchEOLSlurpToken, |
---|
| 411 | ] |
---|
| 412 | |
---|
| 413 | ## regex setup ## |
---|
| 414 | |
---|
| 415 | def _makeCheetahVarREs(self): |
---|
| 416 | |
---|
| 417 | """Setup the regexs for Cheetah $var parsing.""" |
---|
| 418 | |
---|
| 419 | num = r'[0-9\.]+' |
---|
| 420 | interval = (r'(?P<interval>' + |
---|
| 421 | num + r's|' + |
---|
| 422 | num + r'm|' + |
---|
| 423 | num + r'h|' + |
---|
| 424 | num + r'd|' + |
---|
| 425 | num + r'w|' + |
---|
| 426 | num + ')' |
---|
| 427 | ) |
---|
| 428 | |
---|
| 429 | cacheToken = (r'(?:' + |
---|
| 430 | r'(?P<REFRESH_CACHE>\*' + interval + '\*)'+ |
---|
| 431 | '|' + |
---|
| 432 | r'(?P<STATIC_CACHE>\*)' + |
---|
| 433 | '|' + |
---|
| 434 | r'(?P<NO_CACHE>)' + |
---|
| 435 | ')') |
---|
| 436 | self.cacheTokenRE = cachedRegex(cacheToken) |
---|
| 437 | |
---|
| 438 | silentPlaceholderToken = (r'(?:' + |
---|
| 439 | r'(?P<SILENT>' +escapeRegexChars('!')+')'+ |
---|
| 440 | '|' + |
---|
| 441 | r'(?P<NOT_SILENT>)' + |
---|
| 442 | ')') |
---|
| 443 | self.silentPlaceholderTokenRE = cachedRegex(silentPlaceholderToken) |
---|
| 444 | |
---|
| 445 | self.cheetahVarStartRE = cachedRegex( |
---|
| 446 | escCharLookBehind + |
---|
| 447 | r'(?P<startToken>'+escapeRegexChars(self.setting('cheetahVarStartToken'))+')'+ |
---|
| 448 | r'(?P<silenceToken>'+silentPlaceholderToken+')'+ |
---|
| 449 | r'(?P<cacheToken>'+cacheToken+')'+ |
---|
| 450 | r'(?P<enclosure>|(?:(?:\{|\(|\[)[ \t\f]*))' + # allow WS after enclosure |
---|
| 451 | r'(?=[A-Za-z_])') |
---|
| 452 | validCharsLookAhead = r'(?=[A-Za-z_\*!\{\(\[])' |
---|
| 453 | self.cheetahVarStartToken = self.setting('cheetahVarStartToken') |
---|
| 454 | self.cheetahVarStartTokenRE = cachedRegex( |
---|
| 455 | escCharLookBehind + |
---|
| 456 | escapeRegexChars(self.setting('cheetahVarStartToken')) |
---|
| 457 | +validCharsLookAhead |
---|
| 458 | ) |
---|
| 459 | |
---|
| 460 | self.cheetahVarInExpressionStartTokenRE = cachedRegex( |
---|
| 461 | escapeRegexChars(self.setting('cheetahVarStartToken')) |
---|
| 462 | +r'(?=[A-Za-z_])' |
---|
| 463 | ) |
---|
| 464 | |
---|
| 465 | self.expressionPlaceholderStartRE = cachedRegex( |
---|
| 466 | escCharLookBehind + |
---|
| 467 | r'(?P<startToken>' + escapeRegexChars(self.setting('cheetahVarStartToken')) + ')' + |
---|
| 468 | r'(?P<cacheToken>' + cacheToken + ')' + |
---|
| 469 | #r'\[[ \t\f]*' |
---|
| 470 | r'(?:\{|\(|\[)[ \t\f]*' |
---|
| 471 | + r'(?=[^\)\}\]])' |
---|
| 472 | ) |
---|
| 473 | |
---|
| 474 | if self.setting('EOLSlurpToken'): |
---|
| 475 | self.EOLSlurpRE = cachedRegex( |
---|
| 476 | escapeRegexChars(self.setting('EOLSlurpToken')) |
---|
| 477 | + r'[ \t\f]*' |
---|
| 478 | + r'(?:'+EOL+')' |
---|
| 479 | ) |
---|
| 480 | else: |
---|
| 481 | self.EOLSlurpRE = None |
---|
| 482 | |
---|
| 483 | |
---|
| 484 | def _makeCommentREs(self): |
---|
| 485 | """Construct the regex bits that are used in comment parsing.""" |
---|
| 486 | startTokenEsc = escapeRegexChars(self.setting('commentStartToken')) |
---|
| 487 | self.commentStartTokenRE = cachedRegex(escCharLookBehind + startTokenEsc) |
---|
| 488 | del startTokenEsc |
---|
| 489 | |
---|
| 490 | startTokenEsc = escapeRegexChars( |
---|
| 491 | self.setting('multiLineCommentStartToken')) |
---|
| 492 | endTokenEsc = escapeRegexChars( |
---|
| 493 | self.setting('multiLineCommentEndToken')) |
---|
| 494 | self.multiLineCommentTokenStartRE = cachedRegex(escCharLookBehind + |
---|
| 495 | startTokenEsc) |
---|
| 496 | self.multiLineCommentEndTokenRE = cachedRegex(escCharLookBehind + |
---|
| 497 | endTokenEsc) |
---|
| 498 | |
---|
| 499 | def _makeDirectiveREs(self): |
---|
| 500 | """Construct the regexs that are used in directive parsing.""" |
---|
| 501 | startToken = self.setting('directiveStartToken') |
---|
| 502 | endToken = self.setting('directiveEndToken') |
---|
| 503 | startTokenEsc = escapeRegexChars(startToken) |
---|
| 504 | endTokenEsc = escapeRegexChars(endToken) |
---|
| 505 | validSecondCharsLookAhead = r'(?=[A-Za-z_@])' |
---|
| 506 | reParts = [escCharLookBehind, startTokenEsc] |
---|
| 507 | if self.setting('allowWhitespaceAfterDirectiveStartToken'): |
---|
| 508 | reParts.append('[ \t]*') |
---|
| 509 | reParts.append(validSecondCharsLookAhead) |
---|
| 510 | self.directiveStartTokenRE = cachedRegex(''.join(reParts)) |
---|
| 511 | self.directiveEndTokenRE = cachedRegex(escCharLookBehind + endTokenEsc) |
---|
| 512 | |
---|
| 513 | def _makePspREs(self): |
---|
| 514 | """Setup the regexs for PSP parsing.""" |
---|
| 515 | startToken = self.setting('PSPStartToken') |
---|
| 516 | startTokenEsc = escapeRegexChars(startToken) |
---|
| 517 | self.PSPStartTokenRE = cachedRegex(escCharLookBehind + startTokenEsc) |
---|
| 518 | endToken = self.setting('PSPEndToken') |
---|
| 519 | endTokenEsc = escapeRegexChars(endToken) |
---|
| 520 | self.PSPEndTokenRE = cachedRegex(escCharLookBehind + endTokenEsc) |
---|
| 521 | |
---|
| 522 | |
---|
| 523 | def isLineClearToStartToken(self, pos=None): |
---|
| 524 | return self.isLineClearToPos(pos) |
---|
| 525 | |
---|
| 526 | def matchTopLevelToken(self): |
---|
| 527 | """Returns the first match found from the following methods: |
---|
| 528 | self.matchCommentStartToken |
---|
| 529 | self.matchMultiLineCommentStartToken |
---|
| 530 | self.matchVariablePlaceholderStart |
---|
| 531 | self.matchExpressionPlaceholderStart |
---|
| 532 | self.matchDirective |
---|
| 533 | self.matchPSPStartToken |
---|
| 534 | self.matchEOLSlurpToken |
---|
| 535 | |
---|
| 536 | Returns None if no match. |
---|
| 537 | """ |
---|
| 538 | match = None |
---|
| 539 | if self.peek() in self._possibleNonStrConstantChars: |
---|
| 540 | for matcher in self._nonStrConstMatchers: |
---|
| 541 | match = matcher() |
---|
| 542 | if match: |
---|
| 543 | break |
---|
| 544 | return match |
---|
| 545 | |
---|
| 546 | def matchPyToken(self): |
---|
| 547 | match = pseudoprog.match(self.src(), self.pos()) |
---|
| 548 | |
---|
| 549 | if match and match.group() in tripleQuotedStringStarts: |
---|
| 550 | TQSmatch = tripleQuotedStringREs[match.group()].match(self.src(), self.pos()) |
---|
| 551 | if TQSmatch: |
---|
| 552 | return TQSmatch |
---|
| 553 | return match |
---|
| 554 | |
---|
| 555 | def getPyToken(self): |
---|
| 556 | match = self.matchPyToken() |
---|
| 557 | if match is None: |
---|
| 558 | raise ParseError(self) |
---|
| 559 | elif match.group() in tripleQuotedStringStarts: |
---|
| 560 | raise ParseError(self, msg='Malformed triple-quoted string') |
---|
| 561 | return self.readTo(match.end()) |
---|
| 562 | |
---|
| 563 | def matchEOLSlurpToken(self): |
---|
| 564 | if self.EOLSlurpRE: |
---|
| 565 | return self.EOLSlurpRE.match(self.src(), self.pos()) |
---|
| 566 | |
---|
| 567 | def getEOLSlurpToken(self): |
---|
| 568 | match = self.matchEOLSlurpToken() |
---|
| 569 | if not match: |
---|
| 570 | raise ParseError(self, msg='Invalid EOL slurp token') |
---|
| 571 | return self.readTo(match.end()) |
---|
| 572 | |
---|
| 573 | def matchCommentStartToken(self): |
---|
| 574 | return self.commentStartTokenRE.match(self.src(), self.pos()) |
---|
| 575 | |
---|
| 576 | def getCommentStartToken(self): |
---|
| 577 | match = self.matchCommentStartToken() |
---|
| 578 | if not match: |
---|
| 579 | raise ParseError(self, msg='Invalid single-line comment start token') |
---|
| 580 | return self.readTo(match.end()) |
---|
| 581 | |
---|
| 582 | def matchMultiLineCommentStartToken(self): |
---|
| 583 | return self.multiLineCommentTokenStartRE.match(self.src(), self.pos()) |
---|
| 584 | |
---|
| 585 | def getMultiLineCommentStartToken(self): |
---|
| 586 | match = self.matchMultiLineCommentStartToken() |
---|
| 587 | if not match: |
---|
| 588 | raise ParseError(self, msg='Invalid multi-line comment start token') |
---|
| 589 | return self.readTo(match.end()) |
---|
| 590 | |
---|
| 591 | def matchMultiLineCommentEndToken(self): |
---|
| 592 | return self.multiLineCommentEndTokenRE.match(self.src(), self.pos()) |
---|
| 593 | |
---|
| 594 | def getMultiLineCommentEndToken(self): |
---|
| 595 | match = self.matchMultiLineCommentEndToken() |
---|
| 596 | if not match: |
---|
| 597 | raise ParseError(self, msg='Invalid multi-line comment end token') |
---|
| 598 | return self.readTo(match.end()) |
---|
| 599 | |
---|
| 600 | def getCommaSeparatedSymbols(self): |
---|
| 601 | """ |
---|
| 602 | Loosely based on getDottedName to pull out comma separated |
---|
| 603 | named chunks |
---|
| 604 | """ |
---|
| 605 | srcLen = len(self) |
---|
| 606 | pieces = [] |
---|
| 607 | nameChunks = [] |
---|
| 608 | |
---|
| 609 | if not self.peek() in identchars: |
---|
| 610 | raise ParseError(self) |
---|
| 611 | |
---|
| 612 | while self.pos() < srcLen: |
---|
| 613 | c = self.peek() |
---|
| 614 | if c in namechars: |
---|
| 615 | nameChunk = self.getIdentifier() |
---|
| 616 | nameChunks.append(nameChunk) |
---|
| 617 | elif c == '.': |
---|
| 618 | if self.pos()+1 <srcLen and self.peek(1) in identchars: |
---|
| 619 | nameChunks.append(self.getc()) |
---|
| 620 | else: |
---|
| 621 | break |
---|
| 622 | elif c == ',': |
---|
| 623 | self.getc() |
---|
| 624 | pieces.append(''.join(nameChunks)) |
---|
| 625 | nameChunks = [] |
---|
| 626 | elif c in (' ', '\t'): |
---|
| 627 | self.getc() |
---|
| 628 | else: |
---|
| 629 | break |
---|
| 630 | |
---|
| 631 | if nameChunks: |
---|
| 632 | pieces.append(''.join(nameChunks)) |
---|
| 633 | |
---|
| 634 | return pieces |
---|
| 635 | |
---|
| 636 | def getDottedName(self): |
---|
| 637 | srcLen = len(self) |
---|
| 638 | nameChunks = [] |
---|
| 639 | |
---|
| 640 | if not self.peek() in identchars: |
---|
| 641 | raise ParseError(self) |
---|
| 642 | |
---|
| 643 | while self.pos() < srcLen: |
---|
| 644 | c = self.peek() |
---|
| 645 | if c in namechars: |
---|
| 646 | nameChunk = self.getIdentifier() |
---|
| 647 | nameChunks.append(nameChunk) |
---|
| 648 | elif c == '.': |
---|
| 649 | if self.pos()+1 <srcLen and self.peek(1) in identchars: |
---|
| 650 | nameChunks.append(self.getc()) |
---|
| 651 | else: |
---|
| 652 | break |
---|
| 653 | else: |
---|
| 654 | break |
---|
| 655 | |
---|
| 656 | return ''.join(nameChunks) |
---|
| 657 | |
---|
| 658 | def matchIdentifier(self): |
---|
| 659 | return identRE.match(self.src(), self.pos()) |
---|
| 660 | |
---|
| 661 | def getIdentifier(self): |
---|
| 662 | match = self.matchIdentifier() |
---|
| 663 | if not match: |
---|
| 664 | raise ParseError(self, msg='Invalid identifier') |
---|
| 665 | return self.readTo(match.end()) |
---|
| 666 | |
---|
| 667 | def matchOperator(self): |
---|
| 668 | match = self.matchPyToken() |
---|
| 669 | if match and match.group() not in operators: |
---|
| 670 | match = None |
---|
| 671 | return match |
---|
| 672 | |
---|
| 673 | def getOperator(self): |
---|
| 674 | match = self.matchOperator() |
---|
| 675 | if not match: |
---|
| 676 | raise ParseError(self, msg='Expected operator') |
---|
| 677 | return self.readTo( match.end() ) |
---|
| 678 | |
---|
| 679 | def matchAssignmentOperator(self): |
---|
| 680 | match = self.matchPyToken() |
---|
| 681 | if match and match.group() not in assignmentOps: |
---|
| 682 | match = None |
---|
| 683 | return match |
---|
| 684 | |
---|
| 685 | def getAssignmentOperator(self): |
---|
| 686 | match = self.matchAssignmentOperator() |
---|
| 687 | if not match: |
---|
| 688 | raise ParseError(self, msg='Expected assignment operator') |
---|
| 689 | return self.readTo( match.end() ) |
---|
| 690 | |
---|
| 691 | def matchDirective(self): |
---|
| 692 | """Returns False or the name of the directive matched. |
---|
| 693 | """ |
---|
| 694 | startPos = self.pos() |
---|
| 695 | if not self.matchDirectiveStartToken(): |
---|
| 696 | return False |
---|
| 697 | self.getDirectiveStartToken() |
---|
| 698 | directiveName = self.matchDirectiveName() |
---|
| 699 | self.setPos(startPos) |
---|
| 700 | return directiveName |
---|
| 701 | |
---|
| 702 | def matchDirectiveName(self, directiveNameChars=identchars+'0123456789-@'): |
---|
| 703 | startPos = self.pos() |
---|
| 704 | possibleMatches = self._directiveNamesAndParsers.keys() |
---|
| 705 | name = '' |
---|
| 706 | match = None |
---|
| 707 | |
---|
| 708 | while not self.atEnd(): |
---|
| 709 | c = self.getc() |
---|
| 710 | if not c in directiveNameChars: |
---|
| 711 | break |
---|
| 712 | name += c |
---|
| 713 | if name == '@': |
---|
| 714 | if not self.atEnd() and self.peek() in identchars: |
---|
| 715 | match = '@' |
---|
| 716 | break |
---|
| 717 | possibleMatches = [dn for dn in possibleMatches if dn.startswith(name)] |
---|
| 718 | if not possibleMatches: |
---|
| 719 | break |
---|
| 720 | elif (name in possibleMatches and (self.atEnd() or self.peek() not in directiveNameChars)): |
---|
| 721 | match = name |
---|
| 722 | break |
---|
| 723 | |
---|
| 724 | self.setPos(startPos) |
---|
| 725 | return match |
---|
| 726 | |
---|
| 727 | def matchDirectiveStartToken(self): |
---|
| 728 | return self.directiveStartTokenRE.match(self.src(), self.pos()) |
---|
| 729 | |
---|
| 730 | def getDirectiveStartToken(self): |
---|
| 731 | match = self.matchDirectiveStartToken() |
---|
| 732 | if not match: |
---|
| 733 | raise ParseError(self, msg='Invalid directive start token') |
---|
| 734 | return self.readTo(match.end()) |
---|
| 735 | |
---|
| 736 | def matchDirectiveEndToken(self): |
---|
| 737 | return self.directiveEndTokenRE.match(self.src(), self.pos()) |
---|
| 738 | |
---|
| 739 | def getDirectiveEndToken(self): |
---|
| 740 | match = self.matchDirectiveEndToken() |
---|
| 741 | if not match: |
---|
| 742 | raise ParseError(self, msg='Invalid directive end token') |
---|
| 743 | return self.readTo(match.end()) |
---|
| 744 | |
---|
| 745 | |
---|
| 746 | def matchColonForSingleLineShortFormDirective(self): |
---|
| 747 | if not self.atEnd() and self.peek()==':': |
---|
| 748 | restOfLine = self[self.pos()+1:self.findEOL()] |
---|
| 749 | restOfLine = restOfLine.strip() |
---|
| 750 | if not restOfLine: |
---|
| 751 | return False |
---|
| 752 | elif self.commentStartTokenRE.match(restOfLine): |
---|
| 753 | return False |
---|
| 754 | else: # non-whitespace, non-commment chars found |
---|
| 755 | return True |
---|
| 756 | return False |
---|
| 757 | |
---|
| 758 | def matchPSPStartToken(self): |
---|
| 759 | return self.PSPStartTokenRE.match(self.src(), self.pos()) |
---|
| 760 | |
---|
| 761 | def matchPSPEndToken(self): |
---|
| 762 | return self.PSPEndTokenRE.match(self.src(), self.pos()) |
---|
| 763 | |
---|
| 764 | def getPSPStartToken(self): |
---|
| 765 | match = self.matchPSPStartToken() |
---|
| 766 | if not match: |
---|
| 767 | raise ParseError(self, msg='Invalid psp start token') |
---|
| 768 | return self.readTo(match.end()) |
---|
| 769 | |
---|
| 770 | def getPSPEndToken(self): |
---|
| 771 | match = self.matchPSPEndToken() |
---|
| 772 | if not match: |
---|
| 773 | raise ParseError(self, msg='Invalid psp end token') |
---|
| 774 | return self.readTo(match.end()) |
---|
| 775 | |
---|
| 776 | def matchCheetahVarStart(self): |
---|
| 777 | """includes the enclosure and cache token""" |
---|
| 778 | return self.cheetahVarStartRE.match(self.src(), self.pos()) |
---|
| 779 | |
---|
| 780 | def matchCheetahVarStartToken(self): |
---|
| 781 | """includes the enclosure and cache token""" |
---|
| 782 | return self.cheetahVarStartTokenRE.match(self.src(), self.pos()) |
---|
| 783 | |
---|
| 784 | def matchCheetahVarInExpressionStartToken(self): |
---|
| 785 | """no enclosures or cache tokens allowed""" |
---|
| 786 | return self.cheetahVarInExpressionStartTokenRE.match(self.src(), self.pos()) |
---|
| 787 | |
---|
| 788 | def matchVariablePlaceholderStart(self): |
---|
| 789 | """includes the enclosure and cache token""" |
---|
| 790 | return self.cheetahVarStartRE.match(self.src(), self.pos()) |
---|
| 791 | |
---|
| 792 | def matchExpressionPlaceholderStart(self): |
---|
| 793 | """includes the enclosure and cache token""" |
---|
| 794 | return self.expressionPlaceholderStartRE.match(self.src(), self.pos()) |
---|
| 795 | |
---|
| 796 | def getCheetahVarStartToken(self): |
---|
| 797 | """just the start token, not the enclosure or cache token""" |
---|
| 798 | match = self.matchCheetahVarStartToken() |
---|
| 799 | if not match: |
---|
| 800 | raise ParseError(self, msg='Expected Cheetah $var start token') |
---|
| 801 | return self.readTo( match.end() ) |
---|
| 802 | |
---|
| 803 | |
---|
| 804 | def getCacheToken(self): |
---|
| 805 | try: |
---|
| 806 | token = self.cacheTokenRE.match(self.src(), self.pos()) |
---|
| 807 | self.setPos( token.end() ) |
---|
| 808 | return token.group() |
---|
| 809 | except: |
---|
| 810 | raise ParseError(self, msg='Expected cache token') |
---|
| 811 | |
---|
| 812 | def getSilentPlaceholderToken(self): |
---|
| 813 | try: |
---|
| 814 | token = self.silentPlaceholderTokenRE.match(self.src(), self.pos()) |
---|
| 815 | self.setPos( token.end() ) |
---|
| 816 | return token.group() |
---|
| 817 | except: |
---|
| 818 | raise ParseError(self, msg='Expected silent placeholder token') |
---|
| 819 | |
---|
| 820 | |
---|
| 821 | |
---|
| 822 | def getTargetVarsList(self): |
---|
| 823 | varnames = [] |
---|
| 824 | while not self.atEnd(): |
---|
| 825 | if self.peek() in ' \t\f': |
---|
| 826 | self.getWhiteSpace() |
---|
| 827 | elif self.peek() in '\r\n': |
---|
| 828 | break |
---|
| 829 | elif self.startswith(','): |
---|
| 830 | self.advance() |
---|
| 831 | elif self.startswith('in ') or self.startswith('in\t'): |
---|
| 832 | break |
---|
| 833 | #elif self.matchCheetahVarStart(): |
---|
| 834 | elif self.matchCheetahVarInExpressionStartToken(): |
---|
| 835 | self.getCheetahVarStartToken() |
---|
| 836 | self.getSilentPlaceholderToken() |
---|
| 837 | self.getCacheToken() |
---|
| 838 | varnames.append( self.getDottedName() ) |
---|
| 839 | elif self.matchIdentifier(): |
---|
| 840 | varnames.append( self.getDottedName() ) |
---|
| 841 | else: |
---|
| 842 | break |
---|
| 843 | return varnames |
---|
| 844 | |
---|
| 845 | def getCheetahVar(self, plain=False, skipStartToken=False): |
---|
| 846 | """This is called when parsing inside expressions. Cache tokens are only |
---|
| 847 | valid in placeholders so this method discards any cache tokens found. |
---|
| 848 | """ |
---|
| 849 | if not skipStartToken: |
---|
| 850 | self.getCheetahVarStartToken() |
---|
| 851 | self.getSilentPlaceholderToken() |
---|
| 852 | self.getCacheToken() |
---|
| 853 | return self.getCheetahVarBody(plain=plain) |
---|
| 854 | |
---|
| 855 | def getCheetahVarBody(self, plain=False): |
---|
| 856 | # @@TR: this should be in the compiler |
---|
| 857 | return self._compiler.genCheetahVar(self.getCheetahVarNameChunks(), plain=plain) |
---|
| 858 | |
---|
| 859 | def getCheetahVarNameChunks(self): |
---|
| 860 | |
---|
| 861 | """ |
---|
| 862 | nameChunks = list of Cheetah $var subcomponents represented as tuples |
---|
| 863 | [ (namemapperPart,autoCall,restOfName), |
---|
| 864 | ] |
---|
| 865 | where: |
---|
| 866 | namemapperPart = the dottedName base |
---|
| 867 | autocall = where NameMapper should use autocalling on namemapperPart |
---|
| 868 | restOfName = any arglist, index, or slice |
---|
| 869 | |
---|
| 870 | If restOfName contains a call arglist (e.g. '(1234)') then autocall is |
---|
| 871 | False, otherwise it defaults to True. |
---|
| 872 | |
---|
| 873 | EXAMPLE |
---|
| 874 | ------------------------------------------------------------------------ |
---|
| 875 | |
---|
| 876 | if the raw CheetahVar is |
---|
| 877 | $a.b.c[1].d().x.y.z |
---|
| 878 | |
---|
| 879 | nameChunks is the list |
---|
| 880 | [ ('a.b.c',True,'[1]'), |
---|
| 881 | ('d',False,'()'), |
---|
| 882 | ('x.y.z',True,''), |
---|
| 883 | ] |
---|
| 884 | |
---|
| 885 | """ |
---|
| 886 | |
---|
| 887 | chunks = [] |
---|
| 888 | while self.pos() < len(self): |
---|
| 889 | rest = '' |
---|
| 890 | autoCall = True |
---|
| 891 | if not self.peek() in identchars + '.': |
---|
| 892 | break |
---|
| 893 | elif self.peek() == '.': |
---|
| 894 | |
---|
| 895 | if self.pos()+1 < len(self) and self.peek(1) in identchars: |
---|
| 896 | self.advance() # discard the period as it isn't needed with NameMapper |
---|
| 897 | else: |
---|
| 898 | break |
---|
| 899 | |
---|
| 900 | dottedName = self.getDottedName() |
---|
| 901 | if not self.atEnd() and self.peek() in '([': |
---|
| 902 | if self.peek() == '(': |
---|
| 903 | rest = self.getCallArgString() |
---|
| 904 | else: |
---|
| 905 | rest = self.getExpression(enclosed=True) |
---|
| 906 | |
---|
| 907 | period = max(dottedName.rfind('.'), 0) |
---|
| 908 | if period: |
---|
| 909 | chunks.append( (dottedName[:period], autoCall, '') ) |
---|
| 910 | dottedName = dottedName[period+1:] |
---|
| 911 | if rest and rest[0]=='(': |
---|
| 912 | autoCall = False |
---|
| 913 | chunks.append( (dottedName, autoCall, rest) ) |
---|
| 914 | |
---|
| 915 | return chunks |
---|
| 916 | |
---|
| 917 | |
---|
| 918 | def getCallArgString(self, |
---|
| 919 | enclosures=[], # list of tuples (char, pos), where char is ({ or [ |
---|
| 920 | useNameMapper=Unspecified): |
---|
| 921 | |
---|
| 922 | """ Get a method/function call argument string. |
---|
| 923 | |
---|
| 924 | This method understands *arg, and **kw |
---|
| 925 | """ |
---|
| 926 | |
---|
| 927 | # @@TR: this settings mangling should be removed |
---|
| 928 | if useNameMapper is not Unspecified: |
---|
| 929 | useNameMapper_orig = self.setting('useNameMapper') |
---|
| 930 | self.setSetting('useNameMapper', useNameMapper) |
---|
| 931 | |
---|
| 932 | if enclosures: |
---|
| 933 | pass |
---|
| 934 | else: |
---|
| 935 | if not self.peek() == '(': |
---|
| 936 | raise ParseError(self, msg="Expected '('") |
---|
| 937 | startPos = self.pos() |
---|
| 938 | self.getc() |
---|
| 939 | enclosures = [('(', startPos), |
---|
| 940 | ] |
---|
| 941 | |
---|
| 942 | argStringBits = ['('] |
---|
| 943 | addBit = argStringBits.append |
---|
| 944 | |
---|
| 945 | while 1: |
---|
| 946 | if self.atEnd(): |
---|
| 947 | open = enclosures[-1][0] |
---|
| 948 | close = closurePairsRev[open] |
---|
| 949 | self.setPos(enclosures[-1][1]) |
---|
| 950 | raise ParseError( |
---|
| 951 | self, msg="EOF was reached before a matching '" + close + |
---|
| 952 | "' was found for the '" + open + "'") |
---|
| 953 | |
---|
| 954 | c = self.peek() |
---|
| 955 | if c in ")}]": # get the ending enclosure and break |
---|
| 956 | if not enclosures: |
---|
| 957 | raise ParseError(self) |
---|
| 958 | c = self.getc() |
---|
| 959 | open = closurePairs[c] |
---|
| 960 | if enclosures[-1][0] == open: |
---|
| 961 | enclosures.pop() |
---|
| 962 | addBit(')') |
---|
| 963 | break |
---|
| 964 | else: |
---|
| 965 | raise ParseError(self) |
---|
| 966 | elif c in " \t\f\r\n": |
---|
| 967 | addBit(self.getc()) |
---|
| 968 | elif self.matchCheetahVarInExpressionStartToken(): |
---|
| 969 | startPos = self.pos() |
---|
| 970 | codeFor1stToken = self.getCheetahVar() |
---|
| 971 | WS = self.getWhiteSpace() |
---|
| 972 | if not self.atEnd() and self.peek() == '=': |
---|
| 973 | nextToken = self.getPyToken() |
---|
| 974 | if nextToken == '=': |
---|
| 975 | endPos = self.pos() |
---|
| 976 | self.setPos(startPos) |
---|
| 977 | codeFor1stToken = self.getCheetahVar(plain=True) |
---|
| 978 | self.setPos(endPos) |
---|
| 979 | |
---|
| 980 | ## finally |
---|
| 981 | addBit( codeFor1stToken + WS + nextToken ) |
---|
| 982 | else: |
---|
| 983 | addBit( codeFor1stToken + WS) |
---|
| 984 | elif self.matchCheetahVarStart(): |
---|
| 985 | # it has syntax that is only valid at the top level |
---|
| 986 | self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr() |
---|
| 987 | else: |
---|
| 988 | beforeTokenPos = self.pos() |
---|
| 989 | token = self.getPyToken() |
---|
| 990 | if token in ('{','(','['): |
---|
| 991 | self.rev() |
---|
| 992 | token = self.getExpression(enclosed=True) |
---|
| 993 | token = self.transformToken(token, beforeTokenPos) |
---|
| 994 | addBit(token) |
---|
| 995 | |
---|
| 996 | if useNameMapper is not Unspecified: |
---|
| 997 | self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above |
---|
| 998 | |
---|
| 999 | return ''.join(argStringBits) |
---|
| 1000 | |
---|
| 1001 | def getDefArgList(self, exitPos=None, useNameMapper=False): |
---|
| 1002 | |
---|
| 1003 | """ Get an argument list. Can be used for method/function definition |
---|
| 1004 | argument lists or for #directive argument lists. Returns a list of |
---|
| 1005 | tuples in the form (argName, defVal=None) with one tuple for each arg |
---|
| 1006 | name. |
---|
| 1007 | |
---|
| 1008 | These defVals are always strings, so (argName, defVal=None) is safe even |
---|
| 1009 | with a case like (arg1, arg2=None, arg3=1234*2), which would be returned as |
---|
| 1010 | [('arg1', None), |
---|
| 1011 | ('arg2', 'None'), |
---|
| 1012 | ('arg3', '1234*2'), |
---|
| 1013 | ] |
---|
| 1014 | |
---|
| 1015 | This method understands *arg, and **kw |
---|
| 1016 | |
---|
| 1017 | """ |
---|
| 1018 | |
---|
| 1019 | if self.peek() == '(': |
---|
| 1020 | self.advance() |
---|
| 1021 | else: |
---|
| 1022 | exitPos = self.findEOL() # it's a directive so break at the EOL |
---|
| 1023 | argList = ArgList() |
---|
| 1024 | onDefVal = False |
---|
| 1025 | |
---|
| 1026 | # @@TR: this settings mangling should be removed |
---|
| 1027 | useNameMapper_orig = self.setting('useNameMapper') |
---|
| 1028 | self.setSetting('useNameMapper', useNameMapper) |
---|
| 1029 | |
---|
| 1030 | while 1: |
---|
| 1031 | if self.atEnd(): |
---|
| 1032 | raise ParseError( |
---|
| 1033 | self, msg="EOF was reached before a matching ')'"+ |
---|
| 1034 | " was found for the '('") |
---|
| 1035 | |
---|
| 1036 | if self.pos() == exitPos: |
---|
| 1037 | break |
---|
| 1038 | |
---|
| 1039 | c = self.peek() |
---|
| 1040 | if c == ")" or self.matchDirectiveEndToken(): |
---|
| 1041 | break |
---|
| 1042 | elif c == ":": |
---|
| 1043 | break |
---|
| 1044 | elif c in " \t\f\r\n": |
---|
| 1045 | if onDefVal: |
---|
| 1046 | argList.addToDefVal(c) |
---|
| 1047 | self.advance() |
---|
| 1048 | elif c == '=': |
---|
| 1049 | onDefVal = True |
---|
| 1050 | self.advance() |
---|
| 1051 | elif c == ",": |
---|
| 1052 | argList.next() |
---|
| 1053 | onDefVal = False |
---|
| 1054 | self.advance() |
---|
| 1055 | elif self.startswith(self.cheetahVarStartToken) and not onDefVal: |
---|
| 1056 | self.advance(len(self.cheetahVarStartToken)) |
---|
| 1057 | elif self.matchIdentifier() and not onDefVal: |
---|
| 1058 | argList.addArgName( self.getIdentifier() ) |
---|
| 1059 | elif onDefVal: |
---|
| 1060 | if self.matchCheetahVarInExpressionStartToken(): |
---|
| 1061 | token = self.getCheetahVar() |
---|
| 1062 | elif self.matchCheetahVarStart(): |
---|
| 1063 | # it has syntax that is only valid at the top level |
---|
| 1064 | self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr() |
---|
| 1065 | else: |
---|
| 1066 | beforeTokenPos = self.pos() |
---|
| 1067 | token = self.getPyToken() |
---|
| 1068 | if token in ('{','(','['): |
---|
| 1069 | self.rev() |
---|
| 1070 | token = self.getExpression(enclosed=True) |
---|
| 1071 | token = self.transformToken(token, beforeTokenPos) |
---|
| 1072 | argList.addToDefVal(token) |
---|
| 1073 | elif c == '*' and not onDefVal: |
---|
| 1074 | varName = self.getc() |
---|
| 1075 | if self.peek() == '*': |
---|
| 1076 | varName += self.getc() |
---|
| 1077 | if not self.matchIdentifier(): |
---|
| 1078 | raise ParseError(self) |
---|
| 1079 | varName += self.getIdentifier() |
---|
| 1080 | argList.addArgName(varName) |
---|
| 1081 | else: |
---|
| 1082 | raise ParseError(self) |
---|
| 1083 | |
---|
| 1084 | |
---|
| 1085 | self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above |
---|
| 1086 | return argList.merge() |
---|
| 1087 | |
---|
| 1088 | def getExpressionParts(self, |
---|
| 1089 | enclosed=False, |
---|
| 1090 | enclosures=None, # list of tuples (char, pos), where char is ({ or [ |
---|
| 1091 | pyTokensToBreakAt=None, # only works if not enclosed |
---|
| 1092 | useNameMapper=Unspecified, |
---|
| 1093 | ): |
---|
| 1094 | |
---|
| 1095 | """ Get a Cheetah expression that includes $CheetahVars and break at |
---|
| 1096 | directive end tokens, the end of an enclosure, or at a specified |
---|
| 1097 | pyToken. |
---|
| 1098 | """ |
---|
| 1099 | |
---|
| 1100 | if useNameMapper is not Unspecified: |
---|
| 1101 | useNameMapper_orig = self.setting('useNameMapper') |
---|
| 1102 | self.setSetting('useNameMapper', useNameMapper) |
---|
| 1103 | |
---|
| 1104 | if enclosures is None: |
---|
| 1105 | enclosures = [] |
---|
| 1106 | |
---|
| 1107 | srcLen = len(self) |
---|
| 1108 | exprBits = [] |
---|
| 1109 | while 1: |
---|
| 1110 | if self.atEnd(): |
---|
| 1111 | if enclosures: |
---|
| 1112 | open = enclosures[-1][0] |
---|
| 1113 | close = closurePairsRev[open] |
---|
| 1114 | self.setPos(enclosures[-1][1]) |
---|
| 1115 | raise ParseError( |
---|
| 1116 | self, msg="EOF was reached before a matching '" + close + |
---|
| 1117 | "' was found for the '" + open + "'") |
---|
| 1118 | else: |
---|
| 1119 | break |
---|
| 1120 | |
---|
| 1121 | c = self.peek() |
---|
| 1122 | if c in "{([": |
---|
| 1123 | exprBits.append(c) |
---|
| 1124 | enclosures.append( (c, self.pos()) ) |
---|
| 1125 | self.advance() |
---|
| 1126 | elif enclosed and not enclosures: |
---|
| 1127 | break |
---|
| 1128 | elif c in "])}": |
---|
| 1129 | if not enclosures: |
---|
| 1130 | raise ParseError(self) |
---|
| 1131 | open = closurePairs[c] |
---|
| 1132 | if enclosures[-1][0] == open: |
---|
| 1133 | enclosures.pop() |
---|
| 1134 | exprBits.append(c) |
---|
| 1135 | else: |
---|
| 1136 | open = enclosures[-1][0] |
---|
| 1137 | close = closurePairsRev[open] |
---|
| 1138 | row, col = self.getRowCol() |
---|
| 1139 | self.setPos(enclosures[-1][1]) |
---|
| 1140 | raise ParseError( |
---|
| 1141 | self, msg= "A '" + c + "' was found at line " + str(row) + |
---|
| 1142 | ", col " + str(col) + |
---|
| 1143 | " before a matching '" + close + |
---|
| 1144 | "' was found\nfor the '" + open + "'") |
---|
| 1145 | self.advance() |
---|
| 1146 | |
---|
| 1147 | elif c in " \f\t": |
---|
| 1148 | exprBits.append(self.getWhiteSpace()) |
---|
| 1149 | elif self.matchDirectiveEndToken() and not enclosures: |
---|
| 1150 | break |
---|
| 1151 | elif c == "\\" and self.pos()+1 < srcLen: |
---|
| 1152 | eolMatch = EOLre.match(self.src(), self.pos()+1) |
---|
| 1153 | if not eolMatch: |
---|
| 1154 | self.advance() |
---|
| 1155 | raise ParseError(self, msg='Line ending expected') |
---|
| 1156 | self.setPos( eolMatch.end() ) |
---|
| 1157 | elif c in '\r\n': |
---|
| 1158 | if enclosures: |
---|
| 1159 | self.advance() |
---|
| 1160 | else: |
---|
| 1161 | break |
---|
| 1162 | elif self.matchCheetahVarInExpressionStartToken(): |
---|
| 1163 | expr = self.getCheetahVar() |
---|
| 1164 | exprBits.append(expr) |
---|
| 1165 | elif self.matchCheetahVarStart(): |
---|
| 1166 | # it has syntax that is only valid at the top level |
---|
| 1167 | self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr() |
---|
| 1168 | else: |
---|
| 1169 | beforeTokenPos = self.pos() |
---|
| 1170 | token = self.getPyToken() |
---|
| 1171 | if (not enclosures |
---|
| 1172 | and pyTokensToBreakAt |
---|
| 1173 | and token in pyTokensToBreakAt): |
---|
| 1174 | |
---|
| 1175 | self.setPos(beforeTokenPos) |
---|
| 1176 | break |
---|
| 1177 | |
---|
| 1178 | token = self.transformToken(token, beforeTokenPos) |
---|
| 1179 | |
---|
| 1180 | exprBits.append(token) |
---|
| 1181 | if identRE.match(token): |
---|
| 1182 | if token == 'for': |
---|
| 1183 | expr = self.getExpression(useNameMapper=False, pyTokensToBreakAt=['in']) |
---|
| 1184 | exprBits.append(expr) |
---|
| 1185 | else: |
---|
| 1186 | exprBits.append(self.getWhiteSpace()) |
---|
| 1187 | if not self.atEnd() and self.peek() == '(': |
---|
| 1188 | exprBits.append(self.getCallArgString()) |
---|
| 1189 | ## |
---|
| 1190 | if useNameMapper is not Unspecified: |
---|
| 1191 | self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above |
---|
| 1192 | return exprBits |
---|
| 1193 | |
---|
| 1194 | def getExpression(self, |
---|
| 1195 | enclosed=False, |
---|
| 1196 | enclosures=None, # list of tuples (char, pos), where # char is ({ or [ |
---|
| 1197 | pyTokensToBreakAt=None, |
---|
| 1198 | useNameMapper=Unspecified, |
---|
| 1199 | ): |
---|
| 1200 | """Returns the output of self.getExpressionParts() as a concatenated |
---|
| 1201 | string rather than as a list. |
---|
| 1202 | """ |
---|
| 1203 | return ''.join(self.getExpressionParts( |
---|
| 1204 | enclosed=enclosed, enclosures=enclosures, |
---|
| 1205 | pyTokensToBreakAt=pyTokensToBreakAt, |
---|
| 1206 | useNameMapper=useNameMapper)) |
---|
| 1207 | |
---|
| 1208 | |
---|
| 1209 | def transformToken(self, token, beforeTokenPos): |
---|
| 1210 | """Takes a token from the expression being parsed and performs and |
---|
| 1211 | special transformations required by Cheetah. |
---|
| 1212 | |
---|
| 1213 | At the moment only Cheetah's c'$placeholder strings' are transformed. |
---|
| 1214 | """ |
---|
| 1215 | if token=='c' and not self.atEnd() and self.peek() in '\'"': |
---|
| 1216 | nextToken = self.getPyToken() |
---|
| 1217 | token = nextToken.upper() |
---|
| 1218 | theStr = eval(token) |
---|
| 1219 | endPos = self.pos() |
---|
| 1220 | if not theStr: |
---|
| 1221 | return |
---|
| 1222 | |
---|
| 1223 | if token.startswith(single3) or token.startswith(double3): |
---|
| 1224 | startPosIdx = 3 |
---|
| 1225 | else: |
---|
| 1226 | startPosIdx = 1 |
---|
| 1227 | #print 'CHEETAH STRING', nextToken, theStr, startPosIdx |
---|
| 1228 | self.setPos(beforeTokenPos+startPosIdx+1) |
---|
| 1229 | outputExprs = [] |
---|
| 1230 | strConst = '' |
---|
| 1231 | while self.pos() < (endPos-startPosIdx): |
---|
| 1232 | if self.matchCheetahVarStart() or self.matchExpressionPlaceholderStart(): |
---|
| 1233 | if strConst: |
---|
| 1234 | outputExprs.append(repr(strConst)) |
---|
| 1235 | strConst = '' |
---|
| 1236 | placeholderExpr = self.getPlaceholder() |
---|
| 1237 | outputExprs.append('str('+placeholderExpr+')') |
---|
| 1238 | else: |
---|
| 1239 | strConst += self.getc() |
---|
| 1240 | self.setPos(endPos) |
---|
| 1241 | if strConst: |
---|
| 1242 | outputExprs.append(repr(strConst)) |
---|
| 1243 | #if not self.atEnd() and self.matches('.join('): |
---|
| 1244 | # print 'DEBUG***' |
---|
| 1245 | token = "''.join(["+','.join(outputExprs)+"])" |
---|
| 1246 | return token |
---|
| 1247 | |
---|
| 1248 | def _raiseErrorAboutInvalidCheetahVarSyntaxInExpr(self): |
---|
| 1249 | match = self.matchCheetahVarStart() |
---|
| 1250 | groupdict = match.groupdict() |
---|
| 1251 | if groupdict.get('cacheToken'): |
---|
| 1252 | raise ParseError( |
---|
| 1253 | self, |
---|
| 1254 | msg='Cache tokens are not valid inside expressions. ' |
---|
| 1255 | 'Use them in top-level $placeholders only.') |
---|
| 1256 | elif groupdict.get('enclosure'): |
---|
| 1257 | raise ParseError( |
---|
| 1258 | self, |
---|
| 1259 | msg='Long-form placeholders - ${}, $(), $[], etc. are not valid inside expressions. ' |
---|
| 1260 | 'Use them in top-level $placeholders only.') |
---|
| 1261 | else: |
---|
| 1262 | raise ParseError( |
---|
| 1263 | self, |
---|
| 1264 | msg='This form of $placeholder syntax is not valid here.') |
---|
| 1265 | |
---|
| 1266 | |
---|
| 1267 | def getPlaceholder(self, allowCacheTokens=False, plain=False, returnEverything=False): |
---|
| 1268 | # filtered |
---|
| 1269 | for callback in self.setting('preparsePlaceholderHooks'): |
---|
| 1270 | callback(parser=self) |
---|
| 1271 | |
---|
| 1272 | startPos = self.pos() |
---|
| 1273 | lineCol = self.getRowCol(startPos) |
---|
| 1274 | startToken = self.getCheetahVarStartToken() |
---|
| 1275 | silentPlaceholderToken = self.getSilentPlaceholderToken() |
---|
| 1276 | if silentPlaceholderToken: |
---|
| 1277 | isSilentPlaceholder = True |
---|
| 1278 | else: |
---|
| 1279 | isSilentPlaceholder = False |
---|
| 1280 | |
---|
| 1281 | |
---|
| 1282 | if allowCacheTokens: |
---|
| 1283 | cacheToken = self.getCacheToken() |
---|
| 1284 | cacheTokenParts = self.cacheTokenRE.match(cacheToken).groupdict() |
---|
| 1285 | else: |
---|
| 1286 | cacheTokenParts = {} |
---|
| 1287 | |
---|
| 1288 | if self.peek() in '({[': |
---|
| 1289 | pos = self.pos() |
---|
| 1290 | enclosureOpenChar = self.getc() |
---|
| 1291 | enclosures = [ (enclosureOpenChar, pos) ] |
---|
| 1292 | self.getWhiteSpace() |
---|
| 1293 | else: |
---|
| 1294 | enclosures = [] |
---|
| 1295 | |
---|
| 1296 | filterArgs = None |
---|
| 1297 | if self.matchIdentifier(): |
---|
| 1298 | nameChunks = self.getCheetahVarNameChunks() |
---|
| 1299 | expr = self._compiler.genCheetahVar(nameChunks[:], plain=plain) |
---|
| 1300 | restOfExpr = None |
---|
| 1301 | if enclosures: |
---|
| 1302 | WS = self.getWhiteSpace() |
---|
| 1303 | expr += WS |
---|
| 1304 | if self.setting('allowPlaceholderFilterArgs') and self.peek()==',': |
---|
| 1305 | filterArgs = self.getCallArgString(enclosures=enclosures)[1:-1] |
---|
| 1306 | else: |
---|
| 1307 | if self.peek()==closurePairsRev[enclosureOpenChar]: |
---|
| 1308 | self.getc() |
---|
| 1309 | else: |
---|
| 1310 | restOfExpr = self.getExpression(enclosed=True, enclosures=enclosures) |
---|
| 1311 | if restOfExpr[-1] == closurePairsRev[enclosureOpenChar]: |
---|
| 1312 | restOfExpr = restOfExpr[:-1] |
---|
| 1313 | expr += restOfExpr |
---|
| 1314 | rawPlaceholder = self[startPos: self.pos()] |
---|
| 1315 | else: |
---|
| 1316 | expr = self.getExpression(enclosed=True, enclosures=enclosures) |
---|
| 1317 | if expr[-1] == closurePairsRev[enclosureOpenChar]: |
---|
| 1318 | expr = expr[:-1] |
---|
| 1319 | rawPlaceholder=self[startPos: self.pos()] |
---|
| 1320 | |
---|
| 1321 | expr = self._applyExpressionFilters(expr,'placeholder', |
---|
| 1322 | rawExpr=rawPlaceholder,startPos=startPos) |
---|
| 1323 | for callback in self.setting('postparsePlaceholderHooks'): |
---|
| 1324 | callback(parser=self) |
---|
| 1325 | |
---|
| 1326 | if returnEverything: |
---|
| 1327 | return (expr, rawPlaceholder, lineCol, cacheTokenParts, |
---|
| 1328 | filterArgs, isSilentPlaceholder) |
---|
| 1329 | else: |
---|
| 1330 | return expr |
---|
| 1331 | |
---|
| 1332 | |
---|
| 1333 | class _HighLevelParser(_LowLevelParser): |
---|
| 1334 | """This class is a StateMachine for parsing Cheetah source and |
---|
| 1335 | sending state dependent code generation commands to |
---|
| 1336 | Cheetah.Compiler.Compiler. |
---|
| 1337 | """ |
---|
| 1338 | def __init__(self, src, filename=None, breakPoint=None, compiler=None): |
---|
| 1339 | _LowLevelParser.__init__(self, src, filename=filename, breakPoint=breakPoint) |
---|
| 1340 | self.setSettingsManager(compiler) |
---|
| 1341 | self._compiler = compiler |
---|
| 1342 | self.setupState() |
---|
| 1343 | self.configureParser() |
---|
| 1344 | |
---|
| 1345 | def setupState(self): |
---|
| 1346 | self._macros = {} |
---|
| 1347 | self._macroDetails = {} |
---|
| 1348 | self._openDirectivesStack = [] |
---|
| 1349 | |
---|
| 1350 | def cleanup(self): |
---|
| 1351 | """Cleanup to remove any possible reference cycles |
---|
| 1352 | """ |
---|
| 1353 | self._macros.clear() |
---|
| 1354 | for macroname, macroDetails in self._macroDetails.items(): |
---|
| 1355 | macroDetails.template.shutdown() |
---|
| 1356 | del macroDetails.template |
---|
| 1357 | self._macroDetails.clear() |
---|
| 1358 | |
---|
| 1359 | def configureParser(self): |
---|
| 1360 | _LowLevelParser.configureParser(self) |
---|
| 1361 | self._initDirectives() |
---|
| 1362 | |
---|
| 1363 | def _initDirectives(self): |
---|
| 1364 | def normalizeParserVal(val): |
---|
| 1365 | if isinstance(val, (str,unicode)): |
---|
| 1366 | handler = getattr(self, val) |
---|
| 1367 | elif type(val) in (ClassType, TypeType): |
---|
| 1368 | handler = val(self) |
---|
| 1369 | elif callable(val): |
---|
| 1370 | handler = val |
---|
| 1371 | elif val is None: |
---|
| 1372 | handler = val |
---|
| 1373 | else: |
---|
| 1374 | raise Exception('Invalid parser/handler value %r for %s'%(val, name)) |
---|
| 1375 | return handler |
---|
| 1376 | |
---|
| 1377 | normalizeHandlerVal = normalizeParserVal |
---|
| 1378 | |
---|
| 1379 | _directiveNamesAndParsers = directiveNamesAndParsers.copy() |
---|
| 1380 | customNamesAndParsers = self.setting('directiveNamesAndParsers',{}) |
---|
| 1381 | _directiveNamesAndParsers.update(customNamesAndParsers) |
---|
| 1382 | |
---|
| 1383 | _endDirectiveNamesAndHandlers = endDirectiveNamesAndHandlers.copy() |
---|
| 1384 | customNamesAndHandlers = self.setting('endDirectiveNamesAndHandlers',{}) |
---|
| 1385 | _endDirectiveNamesAndHandlers.update(customNamesAndHandlers) |
---|
| 1386 | |
---|
| 1387 | self._directiveNamesAndParsers = {} |
---|
| 1388 | for name, val in _directiveNamesAndParsers.items(): |
---|
| 1389 | if val in (False, 0): |
---|
| 1390 | continue |
---|
| 1391 | self._directiveNamesAndParsers[name] = normalizeParserVal(val) |
---|
| 1392 | |
---|
| 1393 | self._endDirectiveNamesAndHandlers = {} |
---|
| 1394 | for name, val in _endDirectiveNamesAndHandlers.items(): |
---|
| 1395 | if val in (False, 0): |
---|
| 1396 | continue |
---|
| 1397 | self._endDirectiveNamesAndHandlers[name] = normalizeHandlerVal(val) |
---|
| 1398 | |
---|
| 1399 | self._closeableDirectives = ['def','block','closure','defmacro', |
---|
| 1400 | 'call', |
---|
| 1401 | 'capture', |
---|
| 1402 | 'cache', |
---|
| 1403 | 'filter', |
---|
| 1404 | 'if','unless', |
---|
| 1405 | 'for','while','repeat', |
---|
| 1406 | 'try', |
---|
| 1407 | ] |
---|
| 1408 | for directiveName in self.setting('closeableDirectives',[]): |
---|
| 1409 | self._closeableDirectives.append(directiveName) |
---|
| 1410 | |
---|
| 1411 | |
---|
| 1412 | |
---|
| 1413 | macroDirectives = self.setting('macroDirectives',{}) |
---|
| 1414 | macroDirectives['i18n'] = I18n |
---|
| 1415 | |
---|
| 1416 | |
---|
| 1417 | for macroName, callback in macroDirectives.items(): |
---|
| 1418 | if type(callback) in (ClassType, TypeType): |
---|
| 1419 | callback = callback(parser=self) |
---|
| 1420 | assert callback |
---|
| 1421 | self._macros[macroName] = callback |
---|
| 1422 | self._directiveNamesAndParsers[macroName] = self.eatMacroCall |
---|
| 1423 | |
---|
| 1424 | def _applyExpressionFilters(self, expr, exprType, rawExpr=None, startPos=None): |
---|
| 1425 | """Pipes cheetah expressions through a set of optional filter hooks. |
---|
| 1426 | |
---|
| 1427 | The filters are functions which may modify the expressions or raise |
---|
| 1428 | a ForbiddenExpression exception if the expression is not allowed. They |
---|
| 1429 | are defined in the compiler setting 'expressionFilterHooks'. |
---|
| 1430 | |
---|
| 1431 | Some intended use cases: |
---|
| 1432 | |
---|
| 1433 | - to implement 'restricted execution' safeguards in cases where you |
---|
| 1434 | can't trust the author of the template. |
---|
| 1435 | |
---|
| 1436 | - to enforce style guidelines |
---|
| 1437 | |
---|
| 1438 | filter call signature: (parser, expr, exprType, rawExpr=None, startPos=None) |
---|
| 1439 | - parser is the Cheetah parser |
---|
| 1440 | - expr is the expression to filter. In some cases the parser will have |
---|
| 1441 | already modified it from the original source code form. For example, |
---|
| 1442 | placeholders will have been translated into namemapper calls. If you |
---|
| 1443 | need to work with the original source, see rawExpr. |
---|
| 1444 | - exprType is the name of the directive, 'psp', or 'placeholder'. All |
---|
| 1445 | lowercase. @@TR: These will eventually be replaced with a set of |
---|
| 1446 | constants. |
---|
| 1447 | - rawExpr is the original source string that Cheetah parsed. This |
---|
| 1448 | might be None in some cases. |
---|
| 1449 | - startPos is the character position in the source string/file |
---|
| 1450 | where the parser started parsing the current expression. |
---|
| 1451 | |
---|
| 1452 | @@TR: I realize this use of the term 'expression' is a bit wonky as many |
---|
| 1453 | of the 'expressions' are actually statements, but I haven't thought of |
---|
| 1454 | a better name yet. Suggestions? |
---|
| 1455 | """ |
---|
| 1456 | for callback in self.setting('expressionFilterHooks'): |
---|
| 1457 | expr = callback(parser=self, expr=expr, exprType=exprType, |
---|
| 1458 | rawExpr=rawExpr, startPos=startPos) |
---|
| 1459 | return expr |
---|
| 1460 | |
---|
| 1461 | def _filterDisabledDirectives(self, directiveName): |
---|
| 1462 | directiveName = directiveName.lower() |
---|
| 1463 | if (directiveName in self.setting('disabledDirectives') |
---|
| 1464 | or (self.setting('enabledDirectives') |
---|
| 1465 | and directiveName not in self.setting('enabledDirectives'))): |
---|
| 1466 | for callback in self.setting('disabledDirectiveHooks'): |
---|
| 1467 | callback(parser=self, directiveName=directiveName) |
---|
| 1468 | raise ForbiddenDirective(self, msg='This %r directive is disabled'%directiveName) |
---|
| 1469 | |
---|
| 1470 | ## main parse loop |
---|
| 1471 | |
---|
| 1472 | def parse(self, breakPoint=None, assertEmptyStack=True): |
---|
| 1473 | if breakPoint: |
---|
| 1474 | origBP = self.breakPoint() |
---|
| 1475 | self.setBreakPoint(breakPoint) |
---|
| 1476 | assertEmptyStack = False |
---|
| 1477 | |
---|
| 1478 | while not self.atEnd(): |
---|
| 1479 | if self.matchCommentStartToken(): |
---|
| 1480 | self.eatComment() |
---|
| 1481 | elif self.matchMultiLineCommentStartToken(): |
---|
| 1482 | self.eatMultiLineComment() |
---|
| 1483 | elif self.matchVariablePlaceholderStart(): |
---|
| 1484 | self.eatPlaceholder() |
---|
| 1485 | elif self.matchExpressionPlaceholderStart(): |
---|
| 1486 | self.eatPlaceholder() |
---|
| 1487 | elif self.matchDirective(): |
---|
| 1488 | self.eatDirective() |
---|
| 1489 | elif self.matchPSPStartToken(): |
---|
| 1490 | self.eatPSP() |
---|
| 1491 | elif self.matchEOLSlurpToken(): |
---|
| 1492 | self.eatEOLSlurpToken() |
---|
| 1493 | else: |
---|
| 1494 | self.eatPlainText() |
---|
| 1495 | if assertEmptyStack: |
---|
| 1496 | self.assertEmptyOpenDirectivesStack() |
---|
| 1497 | if breakPoint: |
---|
| 1498 | self.setBreakPoint(origBP) |
---|
| 1499 | |
---|
| 1500 | ## non-directive eat methods |
---|
| 1501 | |
---|
| 1502 | def eatPlainText(self): |
---|
| 1503 | startPos = self.pos() |
---|
| 1504 | match = None |
---|
| 1505 | while not self.atEnd(): |
---|
| 1506 | match = self.matchTopLevelToken() |
---|
| 1507 | if match: |
---|
| 1508 | break |
---|
| 1509 | else: |
---|
| 1510 | self.advance() |
---|
| 1511 | strConst = self.readTo(self.pos(), start=startPos) |
---|
| 1512 | self._compiler.addStrConst(strConst) |
---|
| 1513 | return match |
---|
| 1514 | |
---|
| 1515 | def eatComment(self): |
---|
| 1516 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1517 | if isLineClearToStartToken: |
---|
| 1518 | self._compiler.handleWSBeforeDirective() |
---|
| 1519 | self.getCommentStartToken() |
---|
| 1520 | comm = self.readToEOL(gobble=isLineClearToStartToken) |
---|
| 1521 | self._compiler.addComment(comm) |
---|
| 1522 | |
---|
| 1523 | def eatMultiLineComment(self): |
---|
| 1524 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1525 | endOfFirstLine = self.findEOL() |
---|
| 1526 | |
---|
| 1527 | self.getMultiLineCommentStartToken() |
---|
| 1528 | endPos = startPos = self.pos() |
---|
| 1529 | level = 1 |
---|
| 1530 | while 1: |
---|
| 1531 | endPos = self.pos() |
---|
| 1532 | if self.atEnd(): |
---|
| 1533 | break |
---|
| 1534 | if self.matchMultiLineCommentStartToken(): |
---|
| 1535 | self.getMultiLineCommentStartToken() |
---|
| 1536 | level += 1 |
---|
| 1537 | elif self.matchMultiLineCommentEndToken(): |
---|
| 1538 | self.getMultiLineCommentEndToken() |
---|
| 1539 | level -= 1 |
---|
| 1540 | if not level: |
---|
| 1541 | break |
---|
| 1542 | self.advance() |
---|
| 1543 | comm = self.readTo(endPos, start=startPos) |
---|
| 1544 | |
---|
| 1545 | if not self.atEnd(): |
---|
| 1546 | self.getMultiLineCommentEndToken() |
---|
| 1547 | |
---|
| 1548 | if (not self.atEnd()) and self.setting('gobbleWhitespaceAroundMultiLineComments'): |
---|
| 1549 | restOfLine = self[self.pos():self.findEOL()] |
---|
| 1550 | if not restOfLine.strip(): # WS only to EOL |
---|
| 1551 | self.readToEOL(gobble=isLineClearToStartToken) |
---|
| 1552 | |
---|
| 1553 | if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLine): |
---|
| 1554 | self._compiler.handleWSBeforeDirective() |
---|
| 1555 | |
---|
| 1556 | self._compiler.addComment(comm) |
---|
| 1557 | |
---|
| 1558 | def eatPlaceholder(self): |
---|
| 1559 | (expr, rawPlaceholder, |
---|
| 1560 | lineCol, cacheTokenParts, |
---|
| 1561 | filterArgs, isSilentPlaceholder) = self.getPlaceholder( |
---|
| 1562 | allowCacheTokens=True, returnEverything=True) |
---|
| 1563 | |
---|
| 1564 | self._compiler.addPlaceholder( |
---|
| 1565 | expr, |
---|
| 1566 | filterArgs=filterArgs, |
---|
| 1567 | rawPlaceholder=rawPlaceholder, |
---|
| 1568 | cacheTokenParts=cacheTokenParts, |
---|
| 1569 | lineCol=lineCol, |
---|
| 1570 | silentMode=isSilentPlaceholder) |
---|
| 1571 | return |
---|
| 1572 | |
---|
| 1573 | def eatPSP(self): |
---|
| 1574 | # filtered |
---|
| 1575 | self._filterDisabledDirectives(directiveName='psp') |
---|
| 1576 | self.getPSPStartToken() |
---|
| 1577 | endToken = self.setting('PSPEndToken') |
---|
| 1578 | startPos = self.pos() |
---|
| 1579 | while not self.atEnd(): |
---|
| 1580 | if self.peek() == endToken[0]: |
---|
| 1581 | if self.matchPSPEndToken(): |
---|
| 1582 | break |
---|
| 1583 | self.advance() |
---|
| 1584 | pspString = self.readTo(self.pos(), start=startPos).strip() |
---|
| 1585 | pspString = self._applyExpressionFilters(pspString, 'psp', startPos=startPos) |
---|
| 1586 | self._compiler.addPSP(pspString) |
---|
| 1587 | self.getPSPEndToken() |
---|
| 1588 | |
---|
| 1589 | ## generic directive eat methods |
---|
| 1590 | _simpleIndentingDirectives = ''' |
---|
| 1591 | else elif for while repeat unless try except finally'''.split() |
---|
| 1592 | _simpleExprDirectives = ''' |
---|
| 1593 | pass continue stop return yield break |
---|
| 1594 | del assert raise |
---|
| 1595 | silent echo |
---|
| 1596 | import from'''.split() |
---|
| 1597 | _directiveHandlerNames = {'import':'addImportStatement', |
---|
| 1598 | 'from':'addImportStatement', } |
---|
| 1599 | def eatDirective(self): |
---|
| 1600 | directiveName = self.matchDirective() |
---|
| 1601 | self._filterDisabledDirectives(directiveName) |
---|
| 1602 | |
---|
| 1603 | for callback in self.setting('preparseDirectiveHooks'): |
---|
| 1604 | callback(parser=self, directiveName=directiveName) |
---|
| 1605 | |
---|
| 1606 | # subclasses can override the default behaviours here by providing an |
---|
| 1607 | # eater method in self._directiveNamesAndParsers[directiveName] |
---|
| 1608 | directiveParser = self._directiveNamesAndParsers.get(directiveName) |
---|
| 1609 | if directiveParser: |
---|
| 1610 | directiveParser() |
---|
| 1611 | elif directiveName in self._simpleIndentingDirectives: |
---|
| 1612 | handlerName = self._directiveHandlerNames.get(directiveName) |
---|
| 1613 | if not handlerName: |
---|
| 1614 | handlerName = 'add'+directiveName.capitalize() |
---|
| 1615 | handler = getattr(self._compiler, handlerName) |
---|
| 1616 | self.eatSimpleIndentingDirective(directiveName, callback=handler) |
---|
| 1617 | elif directiveName in self._simpleExprDirectives: |
---|
| 1618 | handlerName = self._directiveHandlerNames.get(directiveName) |
---|
| 1619 | if not handlerName: |
---|
| 1620 | handlerName = 'add'+directiveName.capitalize() |
---|
| 1621 | handler = getattr(self._compiler, handlerName) |
---|
| 1622 | if directiveName in ('silent', 'echo'): |
---|
| 1623 | includeDirectiveNameInExpr = False |
---|
| 1624 | else: |
---|
| 1625 | includeDirectiveNameInExpr = True |
---|
| 1626 | expr = self.eatSimpleExprDirective( |
---|
| 1627 | directiveName, |
---|
| 1628 | includeDirectiveNameInExpr=includeDirectiveNameInExpr) |
---|
| 1629 | handler(expr) |
---|
| 1630 | ## |
---|
| 1631 | for callback in self.setting('postparseDirectiveHooks'): |
---|
| 1632 | callback(parser=self, directiveName=directiveName) |
---|
| 1633 | |
---|
| 1634 | def _eatRestOfDirectiveTag(self, isLineClearToStartToken, endOfFirstLinePos): |
---|
| 1635 | foundComment = False |
---|
| 1636 | if self.matchCommentStartToken(): |
---|
| 1637 | pos = self.pos() |
---|
| 1638 | self.advance() |
---|
| 1639 | if not self.matchDirective(): |
---|
| 1640 | self.setPos(pos) |
---|
| 1641 | foundComment = True |
---|
| 1642 | self.eatComment() # this won't gobble the EOL |
---|
| 1643 | else: |
---|
| 1644 | self.setPos(pos) |
---|
| 1645 | |
---|
| 1646 | if not foundComment and self.matchDirectiveEndToken(): |
---|
| 1647 | self.getDirectiveEndToken() |
---|
| 1648 | elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n': |
---|
| 1649 | # still gobble the EOL if a comment was found. |
---|
| 1650 | self.readToEOL(gobble=True) |
---|
| 1651 | |
---|
| 1652 | if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLinePos): |
---|
| 1653 | self._compiler.handleWSBeforeDirective() |
---|
| 1654 | |
---|
| 1655 | def _eatToThisEndDirective(self, directiveName): |
---|
| 1656 | finalPos = endRawPos = startPos = self.pos() |
---|
| 1657 | directiveChar = self.setting('directiveStartToken')[0] |
---|
| 1658 | isLineClearToStartToken = False |
---|
| 1659 | while not self.atEnd(): |
---|
| 1660 | if self.peek() == directiveChar: |
---|
| 1661 | if self.matchDirective() == 'end': |
---|
| 1662 | endRawPos = self.pos() |
---|
| 1663 | self.getDirectiveStartToken() |
---|
| 1664 | self.advance(len('end')) |
---|
| 1665 | self.getWhiteSpace() |
---|
| 1666 | if self.startswith(directiveName): |
---|
| 1667 | if self.isLineClearToStartToken(endRawPos): |
---|
| 1668 | isLineClearToStartToken = True |
---|
| 1669 | endRawPos = self.findBOL(endRawPos) |
---|
| 1670 | self.advance(len(directiveName)) # to end of directiveName |
---|
| 1671 | self.getWhiteSpace() |
---|
| 1672 | finalPos = self.pos() |
---|
| 1673 | break |
---|
| 1674 | self.advance() |
---|
| 1675 | finalPos = endRawPos = self.pos() |
---|
| 1676 | |
---|
| 1677 | textEaten = self.readTo(endRawPos, start=startPos) |
---|
| 1678 | self.setPos(finalPos) |
---|
| 1679 | |
---|
| 1680 | endOfFirstLinePos = self.findEOL() |
---|
| 1681 | |
---|
| 1682 | if self.matchDirectiveEndToken(): |
---|
| 1683 | self.getDirectiveEndToken() |
---|
| 1684 | elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n': |
---|
| 1685 | self.readToEOL(gobble=True) |
---|
| 1686 | |
---|
| 1687 | if isLineClearToStartToken and self.pos() > endOfFirstLinePos: |
---|
| 1688 | self._compiler.handleWSBeforeDirective() |
---|
| 1689 | return textEaten |
---|
| 1690 | |
---|
| 1691 | |
---|
| 1692 | def eatSimpleExprDirective(self, directiveName, includeDirectiveNameInExpr=True): |
---|
| 1693 | # filtered |
---|
| 1694 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1695 | endOfFirstLine = self.findEOL() |
---|
| 1696 | self.getDirectiveStartToken() |
---|
| 1697 | if not includeDirectiveNameInExpr: |
---|
| 1698 | self.advance(len(directiveName)) |
---|
| 1699 | startPos = self.pos() |
---|
| 1700 | expr = self.getExpression().strip() |
---|
| 1701 | directiveName = expr.split()[0] |
---|
| 1702 | expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos) |
---|
| 1703 | if directiveName in self._closeableDirectives: |
---|
| 1704 | self.pushToOpenDirectivesStack(directiveName) |
---|
| 1705 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 1706 | return expr |
---|
| 1707 | |
---|
| 1708 | def eatSimpleIndentingDirective(self, directiveName, callback, |
---|
| 1709 | includeDirectiveNameInExpr=False): |
---|
| 1710 | # filtered |
---|
| 1711 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1712 | endOfFirstLinePos = self.findEOL() |
---|
| 1713 | lineCol = self.getRowCol() |
---|
| 1714 | self.getDirectiveStartToken() |
---|
| 1715 | if directiveName not in 'else elif for while try except finally'.split(): |
---|
| 1716 | self.advance(len(directiveName)) |
---|
| 1717 | startPos = self.pos() |
---|
| 1718 | |
---|
| 1719 | self.getWhiteSpace() |
---|
| 1720 | |
---|
| 1721 | expr = self.getExpression(pyTokensToBreakAt=[':']) |
---|
| 1722 | expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos) |
---|
| 1723 | if self.matchColonForSingleLineShortFormDirective(): |
---|
| 1724 | self.advance() # skip over : |
---|
| 1725 | if directiveName in 'else elif except finally'.split(): |
---|
| 1726 | callback(expr, dedent=False, lineCol=lineCol) |
---|
| 1727 | else: |
---|
| 1728 | callback(expr, lineCol=lineCol) |
---|
| 1729 | |
---|
| 1730 | self.getWhiteSpace(max=1) |
---|
| 1731 | self.parse(breakPoint=self.findEOL(gobble=True)) |
---|
| 1732 | self._compiler.commitStrConst() |
---|
| 1733 | self._compiler.dedent() |
---|
| 1734 | else: |
---|
| 1735 | if self.peek()==':': |
---|
| 1736 | self.advance() |
---|
| 1737 | self.getWhiteSpace() |
---|
| 1738 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 1739 | if directiveName in self._closeableDirectives: |
---|
| 1740 | self.pushToOpenDirectivesStack(directiveName) |
---|
| 1741 | callback(expr, lineCol=lineCol) |
---|
| 1742 | |
---|
| 1743 | def eatEndDirective(self): |
---|
| 1744 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1745 | self.getDirectiveStartToken() |
---|
| 1746 | self.advance(3) # to end of 'end' |
---|
| 1747 | self.getWhiteSpace() |
---|
| 1748 | pos = self.pos() |
---|
| 1749 | directiveName = False |
---|
| 1750 | for key in self._endDirectiveNamesAndHandlers.keys(): |
---|
| 1751 | if self.find(key, pos) == pos: |
---|
| 1752 | directiveName = key |
---|
| 1753 | break |
---|
| 1754 | if not directiveName: |
---|
| 1755 | raise ParseError(self, msg='Invalid end directive') |
---|
| 1756 | |
---|
| 1757 | endOfFirstLinePos = self.findEOL() |
---|
| 1758 | self.getExpression() # eat in any extra comment-like crap |
---|
| 1759 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 1760 | if directiveName in self._closeableDirectives: |
---|
| 1761 | self.popFromOpenDirectivesStack(directiveName) |
---|
| 1762 | |
---|
| 1763 | # subclasses can override the default behaviours here by providing an |
---|
| 1764 | # end-directive handler in self._endDirectiveNamesAndHandlers[directiveName] |
---|
| 1765 | if self._endDirectiveNamesAndHandlers.get(directiveName): |
---|
| 1766 | handler = self._endDirectiveNamesAndHandlers[directiveName] |
---|
| 1767 | handler() |
---|
| 1768 | elif directiveName in 'block capture cache call filter errorCatcher'.split(): |
---|
| 1769 | if key == 'block': |
---|
| 1770 | self._compiler.closeBlock() |
---|
| 1771 | elif key == 'capture': |
---|
| 1772 | self._compiler.endCaptureRegion() |
---|
| 1773 | elif key == 'cache': |
---|
| 1774 | self._compiler.endCacheRegion() |
---|
| 1775 | elif key == 'call': |
---|
| 1776 | self._compiler.endCallRegion() |
---|
| 1777 | elif key == 'filter': |
---|
| 1778 | self._compiler.closeFilterBlock() |
---|
| 1779 | elif key == 'errorCatcher': |
---|
| 1780 | self._compiler.turnErrorCatcherOff() |
---|
| 1781 | elif directiveName in 'while for if try repeat unless'.split(): |
---|
| 1782 | self._compiler.commitStrConst() |
---|
| 1783 | self._compiler.dedent() |
---|
| 1784 | elif directiveName=='closure': |
---|
| 1785 | self._compiler.commitStrConst() |
---|
| 1786 | self._compiler.dedent() |
---|
| 1787 | # @@TR: temporary hack of useSearchList |
---|
| 1788 | self.setSetting('useSearchList', self._useSearchList_orig) |
---|
| 1789 | |
---|
| 1790 | ## specific directive eat methods |
---|
| 1791 | |
---|
| 1792 | def eatBreakPoint(self): |
---|
| 1793 | """Tells the parser to stop parsing at this point and completely ignore |
---|
| 1794 | everything else. |
---|
| 1795 | |
---|
| 1796 | This is a debugging tool. |
---|
| 1797 | """ |
---|
| 1798 | self.setBreakPoint(self.pos()) |
---|
| 1799 | |
---|
| 1800 | def eatShbang(self): |
---|
| 1801 | # filtered |
---|
| 1802 | self.getDirectiveStartToken() |
---|
| 1803 | self.advance(len('shBang')) |
---|
| 1804 | self.getWhiteSpace() |
---|
| 1805 | startPos = self.pos() |
---|
| 1806 | shBang = self.readToEOL() |
---|
| 1807 | shBang = self._applyExpressionFilters(shBang, 'shbang', startPos=startPos) |
---|
| 1808 | self._compiler.setShBang(shBang.strip()) |
---|
| 1809 | |
---|
| 1810 | def eatEncoding(self): |
---|
| 1811 | # filtered |
---|
| 1812 | self.getDirectiveStartToken() |
---|
| 1813 | self.advance(len('encoding')) |
---|
| 1814 | self.getWhiteSpace() |
---|
| 1815 | startPos = self.pos() |
---|
| 1816 | encoding = self.readToEOL() |
---|
| 1817 | encoding = self._applyExpressionFilters(encoding, 'encoding', startPos=startPos) |
---|
| 1818 | self._compiler.setModuleEncoding(encoding.strip()) |
---|
| 1819 | |
---|
| 1820 | def eatCompiler(self): |
---|
| 1821 | # filtered |
---|
| 1822 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1823 | endOfFirstLine = self.findEOL() |
---|
| 1824 | startPos = self.pos() |
---|
| 1825 | self.getDirectiveStartToken() |
---|
| 1826 | self.advance(len('compiler')) # to end of 'compiler' |
---|
| 1827 | self.getWhiteSpace() |
---|
| 1828 | |
---|
| 1829 | startPos = self.pos() |
---|
| 1830 | settingName = self.getIdentifier() |
---|
| 1831 | |
---|
| 1832 | if settingName.lower() == 'reset': |
---|
| 1833 | self.getExpression() # gobble whitespace & junk |
---|
| 1834 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 1835 | self._initializeSettings() |
---|
| 1836 | self.configureParser() |
---|
| 1837 | return |
---|
| 1838 | |
---|
| 1839 | self.getWhiteSpace() |
---|
| 1840 | if self.peek() == '=': |
---|
| 1841 | self.advance() |
---|
| 1842 | else: |
---|
| 1843 | raise ParseError(self) |
---|
| 1844 | valueExpr = self.getExpression() |
---|
| 1845 | endPos = self.pos() |
---|
| 1846 | |
---|
| 1847 | # @@TR: it's unlikely that anyone apply filters would have left this |
---|
| 1848 | # directive enabled: |
---|
| 1849 | # @@TR: fix up filtering, regardless |
---|
| 1850 | self._applyExpressionFilters('%s=%r'%(settingName, valueExpr), |
---|
| 1851 | 'compiler', startPos=startPos) |
---|
| 1852 | |
---|
| 1853 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 1854 | try: |
---|
| 1855 | self._compiler.setCompilerSetting(settingName, valueExpr) |
---|
| 1856 | except: |
---|
| 1857 | out = sys.stderr |
---|
| 1858 | print >> out, 'An error occurred while processing the following #compiler directive.' |
---|
| 1859 | print >> out, '-'*80 |
---|
| 1860 | print >> out, self[startPos:endPos] |
---|
| 1861 | print >> out, '-'*80 |
---|
| 1862 | print >> out, 'Please check the syntax of these settings.' |
---|
| 1863 | print >> out, 'A full Python exception traceback follows.' |
---|
| 1864 | raise |
---|
| 1865 | |
---|
| 1866 | |
---|
| 1867 | def eatCompilerSettings(self): |
---|
| 1868 | # filtered |
---|
| 1869 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1870 | endOfFirstLine = self.findEOL() |
---|
| 1871 | self.getDirectiveStartToken() |
---|
| 1872 | self.advance(len('compiler-settings')) # to end of 'settings' |
---|
| 1873 | |
---|
| 1874 | keywords = self.getTargetVarsList() |
---|
| 1875 | self.getExpression() # gobble any garbage |
---|
| 1876 | |
---|
| 1877 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 1878 | |
---|
| 1879 | if 'reset' in keywords: |
---|
| 1880 | self._compiler._initializeSettings() |
---|
| 1881 | self.configureParser() |
---|
| 1882 | # @@TR: this implies a single-line #compiler-settings directive, and |
---|
| 1883 | # thus we should parse forward for an end directive. |
---|
| 1884 | # Subject to change in the future |
---|
| 1885 | return |
---|
| 1886 | startPos = self.pos() |
---|
| 1887 | settingsStr = self._eatToThisEndDirective('compiler-settings') |
---|
| 1888 | settingsStr = self._applyExpressionFilters(settingsStr, 'compilerSettings', |
---|
| 1889 | startPos=startPos) |
---|
| 1890 | try: |
---|
| 1891 | self._compiler.setCompilerSettings(keywords=keywords, settingsStr=settingsStr) |
---|
| 1892 | except: |
---|
| 1893 | out = sys.stderr |
---|
| 1894 | print >> out, 'An error occurred while processing the following compiler settings.' |
---|
| 1895 | print >> out, '-'*80 |
---|
| 1896 | print >> out, settingsStr.strip() |
---|
| 1897 | print >> out, '-'*80 |
---|
| 1898 | print >> out, 'Please check the syntax of these settings.' |
---|
| 1899 | print >> out, 'A full Python exception traceback follows.' |
---|
| 1900 | raise |
---|
| 1901 | |
---|
| 1902 | def eatAttr(self): |
---|
| 1903 | # filtered |
---|
| 1904 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1905 | endOfFirstLinePos = self.findEOL() |
---|
| 1906 | startPos = self.pos() |
---|
| 1907 | self.getDirectiveStartToken() |
---|
| 1908 | self.advance(len('attr')) |
---|
| 1909 | self.getWhiteSpace() |
---|
| 1910 | startPos = self.pos() |
---|
| 1911 | if self.matchCheetahVarStart(): |
---|
| 1912 | self.getCheetahVarStartToken() |
---|
| 1913 | attribName = self.getIdentifier() |
---|
| 1914 | self.getWhiteSpace() |
---|
| 1915 | self.getAssignmentOperator() |
---|
| 1916 | expr = self.getExpression() |
---|
| 1917 | expr = self._applyExpressionFilters(expr, 'attr', startPos=startPos) |
---|
| 1918 | self._compiler.addAttribute(attribName, expr) |
---|
| 1919 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 1920 | |
---|
| 1921 | def eatDecorator(self): |
---|
| 1922 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1923 | endOfFirstLinePos = self.findEOL() |
---|
| 1924 | startPos = self.pos() |
---|
| 1925 | self.getDirectiveStartToken() |
---|
| 1926 | #self.advance() # eat @ |
---|
| 1927 | startPos = self.pos() |
---|
| 1928 | decoratorExpr = self.getExpression() |
---|
| 1929 | decoratorExpr = self._applyExpressionFilters(decoratorExpr, 'decorator', startPos=startPos) |
---|
| 1930 | self._compiler.addDecorator(decoratorExpr) |
---|
| 1931 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 1932 | self.getWhiteSpace() |
---|
| 1933 | |
---|
| 1934 | directiveName = self.matchDirective() |
---|
| 1935 | if not directiveName or directiveName not in ('def', 'block', 'closure', '@'): |
---|
| 1936 | raise ParseError( |
---|
| 1937 | self, msg='Expected #def, #block, #closure or another @decorator') |
---|
| 1938 | self.eatDirective() |
---|
| 1939 | |
---|
| 1940 | def eatDef(self): |
---|
| 1941 | # filtered |
---|
| 1942 | self._eatDefOrBlock('def') |
---|
| 1943 | |
---|
| 1944 | def eatBlock(self): |
---|
| 1945 | # filtered |
---|
| 1946 | startPos = self.pos() |
---|
| 1947 | methodName, rawSignature = self._eatDefOrBlock('block') |
---|
| 1948 | self._compiler._blockMetaData[methodName] = { |
---|
| 1949 | 'raw':rawSignature, |
---|
| 1950 | 'lineCol':self.getRowCol(startPos), |
---|
| 1951 | } |
---|
| 1952 | |
---|
| 1953 | def eatClosure(self): |
---|
| 1954 | # filtered |
---|
| 1955 | self._eatDefOrBlock('closure') |
---|
| 1956 | |
---|
| 1957 | def _eatDefOrBlock(self, directiveName): |
---|
| 1958 | # filtered |
---|
| 1959 | assert directiveName in ('def','block','closure') |
---|
| 1960 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 1961 | endOfFirstLinePos = self.findEOL() |
---|
| 1962 | startPos = self.pos() |
---|
| 1963 | self.getDirectiveStartToken() |
---|
| 1964 | self.advance(len(directiveName)) |
---|
| 1965 | self.getWhiteSpace() |
---|
| 1966 | if self.matchCheetahVarStart(): |
---|
| 1967 | self.getCheetahVarStartToken() |
---|
| 1968 | methodName = self.getIdentifier() |
---|
| 1969 | self.getWhiteSpace() |
---|
| 1970 | if self.peek() == '(': |
---|
| 1971 | argsList = self.getDefArgList() |
---|
| 1972 | self.advance() # past the closing ')' |
---|
| 1973 | if argsList and argsList[0][0] == 'self': |
---|
| 1974 | del argsList[0] |
---|
| 1975 | else: |
---|
| 1976 | argsList=[] |
---|
| 1977 | |
---|
| 1978 | def includeBlockMarkers(): |
---|
| 1979 | if self.setting('includeBlockMarkers'): |
---|
| 1980 | startMarker = self.setting('blockMarkerStart') |
---|
| 1981 | self._compiler.addStrConst(startMarker[0] + methodName + startMarker[1]) |
---|
| 1982 | |
---|
| 1983 | # @@TR: fix up filtering |
---|
| 1984 | self._applyExpressionFilters(self[startPos:self.pos()], 'def', startPos=startPos) |
---|
| 1985 | |
---|
| 1986 | if self.matchColonForSingleLineShortFormDirective(): |
---|
| 1987 | isNestedDef = (self.setting('allowNestedDefScopes') |
---|
| 1988 | and [name for name in self._openDirectivesStack if name=='def']) |
---|
| 1989 | self.getc() |
---|
| 1990 | rawSignature = self[startPos:endOfFirstLinePos] |
---|
| 1991 | self._eatSingleLineDef(directiveName=directiveName, |
---|
| 1992 | methodName=methodName, |
---|
| 1993 | argsList=argsList, |
---|
| 1994 | startPos=startPos, |
---|
| 1995 | endPos=endOfFirstLinePos) |
---|
| 1996 | if directiveName == 'def' and not isNestedDef: |
---|
| 1997 | #@@TR: must come before _eatRestOfDirectiveTag ... for some reason |
---|
| 1998 | self._compiler.closeDef() |
---|
| 1999 | elif directiveName == 'block': |
---|
| 2000 | includeBlockMarkers() |
---|
| 2001 | self._compiler.closeBlock() |
---|
| 2002 | elif directiveName == 'closure' or isNestedDef: |
---|
| 2003 | self._compiler.dedent() |
---|
| 2004 | |
---|
| 2005 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2006 | else: |
---|
| 2007 | if self.peek()==':': |
---|
| 2008 | self.getc() |
---|
| 2009 | self.pushToOpenDirectivesStack(directiveName) |
---|
| 2010 | rawSignature = self[startPos:self.pos()] |
---|
| 2011 | self._eatMultiLineDef(directiveName=directiveName, |
---|
| 2012 | methodName=methodName, |
---|
| 2013 | argsList=argsList, |
---|
| 2014 | startPos=startPos, |
---|
| 2015 | isLineClearToStartToken=isLineClearToStartToken) |
---|
| 2016 | if directiveName == 'block': |
---|
| 2017 | includeBlockMarkers() |
---|
| 2018 | |
---|
| 2019 | return methodName, rawSignature |
---|
| 2020 | |
---|
| 2021 | def _eatMultiLineDef(self, directiveName, methodName, argsList, startPos, |
---|
| 2022 | isLineClearToStartToken=False): |
---|
| 2023 | # filtered in calling method |
---|
| 2024 | self.getExpression() # slurp up any garbage left at the end |
---|
| 2025 | signature = self[startPos:self.pos()] |
---|
| 2026 | endOfFirstLinePos = self.findEOL() |
---|
| 2027 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2028 | signature = ' '.join([line.strip() for line in signature.splitlines()]) |
---|
| 2029 | parserComment = ('## CHEETAH: generated from ' + signature + |
---|
| 2030 | ' at line %s, col %s' % self.getRowCol(startPos) |
---|
| 2031 | + '.') |
---|
| 2032 | |
---|
| 2033 | isNestedDef = (self.setting('allowNestedDefScopes') |
---|
| 2034 | and len([name for name in self._openDirectivesStack if name=='def'])>1) |
---|
| 2035 | if directiveName=='block' or (directiveName=='def' and not isNestedDef): |
---|
| 2036 | self._compiler.startMethodDef(methodName, argsList, parserComment) |
---|
| 2037 | else: #closure |
---|
| 2038 | self._useSearchList_orig = self.setting('useSearchList') |
---|
| 2039 | self.setSetting('useSearchList', False) |
---|
| 2040 | self._compiler.addClosure(methodName, argsList, parserComment) |
---|
| 2041 | |
---|
| 2042 | return methodName |
---|
| 2043 | |
---|
| 2044 | def _eatSingleLineDef(self, directiveName, methodName, argsList, startPos, endPos): |
---|
| 2045 | # filtered in calling method |
---|
| 2046 | fullSignature = self[startPos:endPos] |
---|
| 2047 | parserComment = ('## Generated from ' + fullSignature + |
---|
| 2048 | ' at line %s, col %s' % self.getRowCol(startPos) |
---|
| 2049 | + '.') |
---|
| 2050 | isNestedDef = (self.setting('allowNestedDefScopes') |
---|
| 2051 | and [name for name in self._openDirectivesStack if name=='def']) |
---|
| 2052 | if directiveName=='block' or (directiveName=='def' and not isNestedDef): |
---|
| 2053 | self._compiler.startMethodDef(methodName, argsList, parserComment) |
---|
| 2054 | else: #closure |
---|
| 2055 | # @@TR: temporary hack of useSearchList |
---|
| 2056 | useSearchList_orig = self.setting('useSearchList') |
---|
| 2057 | self.setSetting('useSearchList', False) |
---|
| 2058 | self._compiler.addClosure(methodName, argsList, parserComment) |
---|
| 2059 | |
---|
| 2060 | self.getWhiteSpace(max=1) |
---|
| 2061 | self.parse(breakPoint=endPos) |
---|
| 2062 | if directiveName=='closure' or isNestedDef: # @@TR: temporary hack of useSearchList |
---|
| 2063 | self.setSetting('useSearchList', useSearchList_orig) |
---|
| 2064 | |
---|
| 2065 | def eatExtends(self): |
---|
| 2066 | # filtered |
---|
| 2067 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2068 | endOfFirstLine = self.findEOL() |
---|
| 2069 | self.getDirectiveStartToken() |
---|
| 2070 | self.advance(len('extends')) |
---|
| 2071 | self.getWhiteSpace() |
---|
| 2072 | startPos = self.pos() |
---|
| 2073 | if self.setting('allowExpressionsInExtendsDirective'): |
---|
| 2074 | baseName = self.getExpression() |
---|
| 2075 | else: |
---|
| 2076 | baseName = self.getCommaSeparatedSymbols() |
---|
| 2077 | baseName = ', '.join(baseName) |
---|
| 2078 | |
---|
| 2079 | baseName = self._applyExpressionFilters(baseName, 'extends', startPos=startPos) |
---|
| 2080 | self._compiler.setBaseClass(baseName) # in compiler |
---|
| 2081 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 2082 | |
---|
| 2083 | def eatImplements(self): |
---|
| 2084 | # filtered |
---|
| 2085 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2086 | endOfFirstLine = self.findEOL() |
---|
| 2087 | self.getDirectiveStartToken() |
---|
| 2088 | self.advance(len('implements')) |
---|
| 2089 | self.getWhiteSpace() |
---|
| 2090 | startPos = self.pos() |
---|
| 2091 | methodName = self.getIdentifier() |
---|
| 2092 | if not self.atEnd() and self.peek() == '(': |
---|
| 2093 | argsList = self.getDefArgList() |
---|
| 2094 | self.advance() # past the closing ')' |
---|
| 2095 | if argsList and argsList[0][0] == 'self': |
---|
| 2096 | del argsList[0] |
---|
| 2097 | else: |
---|
| 2098 | argsList=[] |
---|
| 2099 | |
---|
| 2100 | # @@TR: need to split up filtering of the methodname and the args |
---|
| 2101 | #methodName = self._applyExpressionFilters(methodName, 'implements', startPos=startPos) |
---|
| 2102 | self._applyExpressionFilters(self[startPos:self.pos()], 'implements', startPos=startPos) |
---|
| 2103 | |
---|
| 2104 | self._compiler.setMainMethodName(methodName) |
---|
| 2105 | self._compiler.setMainMethodArgs(argsList) |
---|
| 2106 | |
---|
| 2107 | self.getExpression() # throw away and unwanted crap that got added in |
---|
| 2108 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 2109 | |
---|
| 2110 | def eatSuper(self): |
---|
| 2111 | # filtered |
---|
| 2112 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2113 | endOfFirstLine = self.findEOL() |
---|
| 2114 | self.getDirectiveStartToken() |
---|
| 2115 | self.advance(len('super')) |
---|
| 2116 | self.getWhiteSpace() |
---|
| 2117 | startPos = self.pos() |
---|
| 2118 | if not self.atEnd() and self.peek() == '(': |
---|
| 2119 | argsList = self.getDefArgList() |
---|
| 2120 | self.advance() # past the closing ')' |
---|
| 2121 | if argsList and argsList[0][0] == 'self': |
---|
| 2122 | del argsList[0] |
---|
| 2123 | else: |
---|
| 2124 | argsList=[] |
---|
| 2125 | |
---|
| 2126 | self._applyExpressionFilters(self[startPos:self.pos()], 'super', startPos=startPos) |
---|
| 2127 | |
---|
| 2128 | #parserComment = ('## CHEETAH: generated from ' + signature + |
---|
| 2129 | # ' at line %s, col %s' % self.getRowCol(startPos) |
---|
| 2130 | # + '.') |
---|
| 2131 | |
---|
| 2132 | self.getExpression() # throw away and unwanted crap that got added in |
---|
| 2133 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 2134 | self._compiler.addSuper(argsList) |
---|
| 2135 | |
---|
| 2136 | def eatSet(self): |
---|
| 2137 | # filtered |
---|
| 2138 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2139 | endOfFirstLine = self.findEOL() |
---|
| 2140 | self.getDirectiveStartToken() |
---|
| 2141 | self.advance(3) |
---|
| 2142 | self.getWhiteSpace() |
---|
| 2143 | style = SET_LOCAL |
---|
| 2144 | if self.startswith('local'): |
---|
| 2145 | self.getIdentifier() |
---|
| 2146 | self.getWhiteSpace() |
---|
| 2147 | elif self.startswith('global'): |
---|
| 2148 | self.getIdentifier() |
---|
| 2149 | self.getWhiteSpace() |
---|
| 2150 | style = SET_GLOBAL |
---|
| 2151 | elif self.startswith('module'): |
---|
| 2152 | self.getIdentifier() |
---|
| 2153 | self.getWhiteSpace() |
---|
| 2154 | style = SET_MODULE |
---|
| 2155 | |
---|
| 2156 | startsWithDollar = self.matchCheetahVarStart() |
---|
| 2157 | startPos = self.pos() |
---|
| 2158 | LVALUE = self.getExpression(pyTokensToBreakAt=assignmentOps, useNameMapper=False).strip() |
---|
| 2159 | OP = self.getAssignmentOperator() |
---|
| 2160 | RVALUE = self.getExpression() |
---|
| 2161 | expr = LVALUE + ' ' + OP + ' ' + RVALUE.strip() |
---|
| 2162 | |
---|
| 2163 | expr = self._applyExpressionFilters(expr, 'set', startPos=startPos) |
---|
| 2164 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 2165 | |
---|
| 2166 | class Components: pass # used for 'set global' |
---|
| 2167 | exprComponents = Components() |
---|
| 2168 | exprComponents.LVALUE = LVALUE |
---|
| 2169 | exprComponents.OP = OP |
---|
| 2170 | exprComponents.RVALUE = RVALUE |
---|
| 2171 | self._compiler.addSet(expr, exprComponents, style) |
---|
| 2172 | |
---|
| 2173 | def eatSlurp(self): |
---|
| 2174 | if self.isLineClearToStartToken(): |
---|
| 2175 | self._compiler.handleWSBeforeDirective() |
---|
| 2176 | self._compiler.commitStrConst() |
---|
| 2177 | self.readToEOL(gobble=True) |
---|
| 2178 | |
---|
| 2179 | def eatEOLSlurpToken(self): |
---|
| 2180 | if self.isLineClearToStartToken(): |
---|
| 2181 | self._compiler.handleWSBeforeDirective() |
---|
| 2182 | self._compiler.commitStrConst() |
---|
| 2183 | self.readToEOL(gobble=True) |
---|
| 2184 | |
---|
| 2185 | def eatRaw(self): |
---|
| 2186 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2187 | endOfFirstLinePos = self.findEOL() |
---|
| 2188 | self.getDirectiveStartToken() |
---|
| 2189 | self.advance(len('raw')) |
---|
| 2190 | self.getWhiteSpace() |
---|
| 2191 | if self.matchColonForSingleLineShortFormDirective(): |
---|
| 2192 | self.advance() # skip over : |
---|
| 2193 | self.getWhiteSpace(max=1) |
---|
| 2194 | rawBlock = self.readToEOL(gobble=False) |
---|
| 2195 | else: |
---|
| 2196 | if self.peek()==':': |
---|
| 2197 | self.advance() |
---|
| 2198 | self.getWhiteSpace() |
---|
| 2199 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2200 | rawBlock = self._eatToThisEndDirective('raw') |
---|
| 2201 | self._compiler.addRawText(rawBlock) |
---|
| 2202 | |
---|
| 2203 | def eatInclude(self): |
---|
| 2204 | # filtered |
---|
| 2205 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2206 | endOfFirstLinePos = self.findEOL() |
---|
| 2207 | self.getDirectiveStartToken() |
---|
| 2208 | self.advance(len('include')) |
---|
| 2209 | |
---|
| 2210 | self.getWhiteSpace() |
---|
| 2211 | includeFrom = 'file' |
---|
| 2212 | isRaw = False |
---|
| 2213 | if self.startswith('raw'): |
---|
| 2214 | self.advance(3) |
---|
| 2215 | isRaw=True |
---|
| 2216 | |
---|
| 2217 | self.getWhiteSpace() |
---|
| 2218 | if self.startswith('source'): |
---|
| 2219 | self.advance(len('source')) |
---|
| 2220 | includeFrom = 'str' |
---|
| 2221 | self.getWhiteSpace() |
---|
| 2222 | if not self.peek() == '=': |
---|
| 2223 | raise ParseError(self) |
---|
| 2224 | self.advance() |
---|
| 2225 | startPos = self.pos() |
---|
| 2226 | sourceExpr = self.getExpression() |
---|
| 2227 | sourceExpr = self._applyExpressionFilters(sourceExpr, 'include', startPos=startPos) |
---|
| 2228 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2229 | self._compiler.addInclude(sourceExpr, includeFrom, isRaw) |
---|
| 2230 | |
---|
| 2231 | |
---|
| 2232 | def eatDefMacro(self): |
---|
| 2233 | # @@TR: not filtered yet |
---|
| 2234 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2235 | endOfFirstLinePos = self.findEOL() |
---|
| 2236 | self.getDirectiveStartToken() |
---|
| 2237 | self.advance(len('defmacro')) |
---|
| 2238 | |
---|
| 2239 | self.getWhiteSpace() |
---|
| 2240 | if self.matchCheetahVarStart(): |
---|
| 2241 | self.getCheetahVarStartToken() |
---|
| 2242 | macroName = self.getIdentifier() |
---|
| 2243 | self.getWhiteSpace() |
---|
| 2244 | if self.peek() == '(': |
---|
| 2245 | argsList = self.getDefArgList(useNameMapper=False) |
---|
| 2246 | self.advance() # past the closing ')' |
---|
| 2247 | if argsList and argsList[0][0] == 'self': |
---|
| 2248 | del argsList[0] |
---|
| 2249 | else: |
---|
| 2250 | argsList=[] |
---|
| 2251 | |
---|
| 2252 | assert not self._directiveNamesAndParsers.has_key(macroName) |
---|
| 2253 | argsList.insert(0, ('src',None)) |
---|
| 2254 | argsList.append(('parser','None')) |
---|
| 2255 | argsList.append(('macros','None')) |
---|
| 2256 | argsList.append(('compilerSettings','None')) |
---|
| 2257 | argsList.append(('isShortForm','None')) |
---|
| 2258 | argsList.append(('EOLCharsInShortForm','None')) |
---|
| 2259 | argsList.append(('startPos','None')) |
---|
| 2260 | argsList.append(('endPos','None')) |
---|
| 2261 | |
---|
| 2262 | if self.matchColonForSingleLineShortFormDirective(): |
---|
| 2263 | self.advance() # skip over : |
---|
| 2264 | self.getWhiteSpace(max=1) |
---|
| 2265 | macroSrc = self.readToEOL(gobble=False) |
---|
| 2266 | self.readToEOL(gobble=True) |
---|
| 2267 | else: |
---|
| 2268 | if self.peek()==':': |
---|
| 2269 | self.advance() |
---|
| 2270 | self.getWhiteSpace() |
---|
| 2271 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2272 | macroSrc = self._eatToThisEndDirective('defmacro') |
---|
| 2273 | |
---|
| 2274 | #print argsList |
---|
| 2275 | normalizedMacroSrc = ''.join( |
---|
| 2276 | ['%def callMacro('+','.join([defv and '%s=%s'%(n,defv) or n |
---|
| 2277 | for n,defv in argsList]) |
---|
| 2278 | +')\n', |
---|
| 2279 | macroSrc, |
---|
| 2280 | '%end def']) |
---|
| 2281 | |
---|
| 2282 | |
---|
| 2283 | from Cheetah.Template import Template |
---|
| 2284 | templateAPIClass = self.setting('templateAPIClassForDefMacro', default=Template) |
---|
| 2285 | compilerSettings = self.setting('compilerSettingsForDefMacro', default={}) |
---|
| 2286 | searchListForMacros = self.setting('searchListForDefMacro', default=[]) |
---|
| 2287 | searchListForMacros = list(searchListForMacros) # copy to avoid mutation bugs |
---|
| 2288 | searchListForMacros.append({'macros':self._macros, |
---|
| 2289 | 'parser':self, |
---|
| 2290 | 'compilerSettings':self.settings(), |
---|
| 2291 | }) |
---|
| 2292 | |
---|
| 2293 | templateAPIClass._updateSettingsWithPreprocessTokens( |
---|
| 2294 | compilerSettings, placeholderToken='@', directiveToken='%') |
---|
| 2295 | macroTemplateClass = templateAPIClass.compile(source=normalizedMacroSrc, |
---|
| 2296 | compilerSettings=compilerSettings) |
---|
| 2297 | #print normalizedMacroSrc |
---|
| 2298 | #t = macroTemplateClass() |
---|
| 2299 | #print t.callMacro('src') |
---|
| 2300 | #print t.generatedClassCode() |
---|
| 2301 | |
---|
| 2302 | class MacroDetails: pass |
---|
| 2303 | macroDetails = MacroDetails() |
---|
| 2304 | macroDetails.macroSrc = macroSrc |
---|
| 2305 | macroDetails.argsList = argsList |
---|
| 2306 | macroDetails.template = macroTemplateClass(searchList=searchListForMacros) |
---|
| 2307 | |
---|
| 2308 | self._macroDetails[macroName] = macroDetails |
---|
| 2309 | self._macros[macroName] = macroDetails.template.callMacro |
---|
| 2310 | self._directiveNamesAndParsers[macroName] = self.eatMacroCall |
---|
| 2311 | |
---|
| 2312 | def eatMacroCall(self): |
---|
| 2313 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2314 | endOfFirstLinePos = self.findEOL() |
---|
| 2315 | startPos = self.pos() |
---|
| 2316 | self.getDirectiveStartToken() |
---|
| 2317 | macroName = self.getIdentifier() |
---|
| 2318 | macro = self._macros[macroName] |
---|
| 2319 | if hasattr(macro, 'parse'): |
---|
| 2320 | return macro.parse(parser=self, startPos=startPos) |
---|
| 2321 | |
---|
| 2322 | if hasattr(macro, 'parseArgs'): |
---|
| 2323 | args = macro.parseArgs(parser=self, startPos=startPos) |
---|
| 2324 | else: |
---|
| 2325 | self.getWhiteSpace() |
---|
| 2326 | args = self.getExpression(useNameMapper=False, |
---|
| 2327 | pyTokensToBreakAt=[':']).strip() |
---|
| 2328 | |
---|
| 2329 | if self.matchColonForSingleLineShortFormDirective(): |
---|
| 2330 | isShortForm = True |
---|
| 2331 | self.advance() # skip over : |
---|
| 2332 | self.getWhiteSpace(max=1) |
---|
| 2333 | srcBlock = self.readToEOL(gobble=False) |
---|
| 2334 | EOLCharsInShortForm = self.readToEOL(gobble=True) |
---|
| 2335 | #self.readToEOL(gobble=False) |
---|
| 2336 | else: |
---|
| 2337 | isShortForm = False |
---|
| 2338 | if self.peek()==':': |
---|
| 2339 | self.advance() |
---|
| 2340 | self.getWhiteSpace() |
---|
| 2341 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2342 | srcBlock = self._eatToThisEndDirective(macroName) |
---|
| 2343 | |
---|
| 2344 | |
---|
| 2345 | if hasattr(macro, 'convertArgStrToDict'): |
---|
| 2346 | kwArgs = macro.convertArgStrToDict(args, parser=self, startPos=startPos) |
---|
| 2347 | else: |
---|
| 2348 | def getArgs(*pargs, **kws): |
---|
| 2349 | return pargs, kws |
---|
| 2350 | exec 'positionalArgs, kwArgs = getArgs(%(args)s)'%locals() |
---|
| 2351 | |
---|
| 2352 | assert not kwArgs.has_key('src') |
---|
| 2353 | kwArgs['src'] = srcBlock |
---|
| 2354 | |
---|
| 2355 | if type(macro)==new.instancemethod: |
---|
| 2356 | co = macro.im_func.func_code |
---|
| 2357 | elif (hasattr(macro, '__call__') |
---|
| 2358 | and hasattr(macro.__call__, 'im_func')): |
---|
| 2359 | co = macro.__call__.im_func.func_code |
---|
| 2360 | else: |
---|
| 2361 | co = macro.func_code |
---|
| 2362 | availableKwArgs = inspect.getargs(co)[0] |
---|
| 2363 | |
---|
| 2364 | if 'parser' in availableKwArgs: |
---|
| 2365 | kwArgs['parser'] = self |
---|
| 2366 | if 'macros' in availableKwArgs: |
---|
| 2367 | kwArgs['macros'] = self._macros |
---|
| 2368 | if 'compilerSettings' in availableKwArgs: |
---|
| 2369 | kwArgs['compilerSettings'] = self.settings() |
---|
| 2370 | if 'isShortForm' in availableKwArgs: |
---|
| 2371 | kwArgs['isShortForm'] = isShortForm |
---|
| 2372 | if isShortForm and 'EOLCharsInShortForm' in availableKwArgs: |
---|
| 2373 | kwArgs['EOLCharsInShortForm'] = EOLCharsInShortForm |
---|
| 2374 | |
---|
| 2375 | if 'startPos' in availableKwArgs: |
---|
| 2376 | kwArgs['startPos'] = startPos |
---|
| 2377 | if 'endPos' in availableKwArgs: |
---|
| 2378 | kwArgs['endPos'] = self.pos() |
---|
| 2379 | |
---|
| 2380 | srcFromMacroOutput = macro(**kwArgs) |
---|
| 2381 | |
---|
| 2382 | origParseSrc = self._src |
---|
| 2383 | origBreakPoint = self.breakPoint() |
---|
| 2384 | origPos = self.pos() |
---|
| 2385 | # add a comment to the output about the macro src that is being parsed |
---|
| 2386 | # or add a comment prefix to all the comments added by the compiler |
---|
| 2387 | self._src = srcFromMacroOutput |
---|
| 2388 | self.setPos(0) |
---|
| 2389 | self.setBreakPoint(len(srcFromMacroOutput)) |
---|
| 2390 | |
---|
| 2391 | self.parse(assertEmptyStack=False) |
---|
| 2392 | |
---|
| 2393 | self._src = origParseSrc |
---|
| 2394 | self.setBreakPoint(origBreakPoint) |
---|
| 2395 | self.setPos(origPos) |
---|
| 2396 | |
---|
| 2397 | |
---|
| 2398 | #self._compiler.addRawText('end') |
---|
| 2399 | |
---|
| 2400 | def eatCache(self): |
---|
| 2401 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2402 | endOfFirstLinePos = self.findEOL() |
---|
| 2403 | lineCol = self.getRowCol() |
---|
| 2404 | self.getDirectiveStartToken() |
---|
| 2405 | self.advance(len('cache')) |
---|
| 2406 | |
---|
| 2407 | startPos = self.pos() |
---|
| 2408 | argList = self.getDefArgList(useNameMapper=True) |
---|
| 2409 | argList = self._applyExpressionFilters(argList, 'cache', startPos=startPos) |
---|
| 2410 | |
---|
| 2411 | def startCache(): |
---|
| 2412 | cacheInfo = self._compiler.genCacheInfoFromArgList(argList) |
---|
| 2413 | self._compiler.startCacheRegion(cacheInfo, lineCol) |
---|
| 2414 | |
---|
| 2415 | if self.matchColonForSingleLineShortFormDirective(): |
---|
| 2416 | self.advance() # skip over : |
---|
| 2417 | self.getWhiteSpace(max=1) |
---|
| 2418 | startCache() |
---|
| 2419 | self.parse(breakPoint=self.findEOL(gobble=True)) |
---|
| 2420 | self._compiler.endCacheRegion() |
---|
| 2421 | else: |
---|
| 2422 | if self.peek()==':': |
---|
| 2423 | self.advance() |
---|
| 2424 | self.getWhiteSpace() |
---|
| 2425 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2426 | self.pushToOpenDirectivesStack('cache') |
---|
| 2427 | startCache() |
---|
| 2428 | |
---|
| 2429 | def eatCall(self): |
---|
| 2430 | # @@TR: need to enable single line version of this |
---|
| 2431 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2432 | endOfFirstLinePos = self.findEOL() |
---|
| 2433 | lineCol = self.getRowCol() |
---|
| 2434 | self.getDirectiveStartToken() |
---|
| 2435 | self.advance(len('call')) |
---|
| 2436 | startPos = self.pos() |
---|
| 2437 | |
---|
| 2438 | useAutocallingOrig = self.setting('useAutocalling') |
---|
| 2439 | self.setSetting('useAutocalling', False) |
---|
| 2440 | self.getWhiteSpace() |
---|
| 2441 | if self.matchCheetahVarStart(): |
---|
| 2442 | functionName = self.getCheetahVar() |
---|
| 2443 | else: |
---|
| 2444 | functionName = self.getCheetahVar(plain=True, skipStartToken=True) |
---|
| 2445 | self.setSetting('useAutocalling', useAutocallingOrig) |
---|
| 2446 | # @@TR: fix up filtering |
---|
| 2447 | self._applyExpressionFilters(self[startPos:self.pos()], 'call', startPos=startPos) |
---|
| 2448 | |
---|
| 2449 | self.getWhiteSpace() |
---|
| 2450 | args = self.getExpression(pyTokensToBreakAt=[':']).strip() |
---|
| 2451 | if self.matchColonForSingleLineShortFormDirective(): |
---|
| 2452 | self.advance() # skip over : |
---|
| 2453 | self._compiler.startCallRegion(functionName, args, lineCol) |
---|
| 2454 | self.getWhiteSpace(max=1) |
---|
| 2455 | self.parse(breakPoint=self.findEOL(gobble=False)) |
---|
| 2456 | self._compiler.endCallRegion() |
---|
| 2457 | else: |
---|
| 2458 | if self.peek()==':': |
---|
| 2459 | self.advance() |
---|
| 2460 | self.getWhiteSpace() |
---|
| 2461 | self.pushToOpenDirectivesStack("call") |
---|
| 2462 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2463 | self._compiler.startCallRegion(functionName, args, lineCol) |
---|
| 2464 | |
---|
| 2465 | def eatCallArg(self): |
---|
| 2466 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2467 | endOfFirstLinePos = self.findEOL() |
---|
| 2468 | lineCol = self.getRowCol() |
---|
| 2469 | self.getDirectiveStartToken() |
---|
| 2470 | |
---|
| 2471 | self.advance(len('arg')) |
---|
| 2472 | startPos = self.pos() |
---|
| 2473 | self.getWhiteSpace() |
---|
| 2474 | argName = self.getIdentifier() |
---|
| 2475 | self.getWhiteSpace() |
---|
| 2476 | argName = self._applyExpressionFilters(argName, 'arg', startPos=startPos) |
---|
| 2477 | self._compiler.setCallArg(argName, lineCol) |
---|
| 2478 | if self.peek() == ':': |
---|
| 2479 | self.getc() |
---|
| 2480 | else: |
---|
| 2481 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2482 | |
---|
| 2483 | def eatFilter(self): |
---|
| 2484 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2485 | endOfFirstLinePos = self.findEOL() |
---|
| 2486 | |
---|
| 2487 | self.getDirectiveStartToken() |
---|
| 2488 | self.advance(len('filter')) |
---|
| 2489 | self.getWhiteSpace() |
---|
| 2490 | startPos = self.pos() |
---|
| 2491 | if self.matchCheetahVarStart(): |
---|
| 2492 | isKlass = True |
---|
| 2493 | theFilter = self.getExpression(pyTokensToBreakAt=[':']) |
---|
| 2494 | else: |
---|
| 2495 | isKlass = False |
---|
| 2496 | theFilter = self.getIdentifier() |
---|
| 2497 | self.getWhiteSpace() |
---|
| 2498 | theFilter = self._applyExpressionFilters(theFilter, 'filter', startPos=startPos) |
---|
| 2499 | |
---|
| 2500 | if self.matchColonForSingleLineShortFormDirective(): |
---|
| 2501 | self.advance() # skip over : |
---|
| 2502 | self.getWhiteSpace(max=1) |
---|
| 2503 | self._compiler.setFilter(theFilter, isKlass) |
---|
| 2504 | self.parse(breakPoint=self.findEOL(gobble=False)) |
---|
| 2505 | self._compiler.closeFilterBlock() |
---|
| 2506 | else: |
---|
| 2507 | if self.peek()==':': |
---|
| 2508 | self.advance() |
---|
| 2509 | self.getWhiteSpace() |
---|
| 2510 | self.pushToOpenDirectivesStack("filter") |
---|
| 2511 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2512 | self._compiler.setFilter(theFilter, isKlass) |
---|
| 2513 | |
---|
| 2514 | def eatTransform(self): |
---|
| 2515 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2516 | endOfFirstLinePos = self.findEOL() |
---|
| 2517 | |
---|
| 2518 | self.getDirectiveStartToken() |
---|
| 2519 | self.advance(len('transform')) |
---|
| 2520 | self.getWhiteSpace() |
---|
| 2521 | startPos = self.pos() |
---|
| 2522 | if self.matchCheetahVarStart(): |
---|
| 2523 | isKlass = True |
---|
| 2524 | transformer = self.getExpression(pyTokensToBreakAt=[':']) |
---|
| 2525 | else: |
---|
| 2526 | isKlass = False |
---|
| 2527 | transformer = self.getIdentifier() |
---|
| 2528 | self.getWhiteSpace() |
---|
| 2529 | transformer = self._applyExpressionFilters(transformer, 'transform', startPos=startPos) |
---|
| 2530 | |
---|
| 2531 | if self.peek()==':': |
---|
| 2532 | self.advance() |
---|
| 2533 | self.getWhiteSpace() |
---|
| 2534 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2535 | self._compiler.setTransform(transformer, isKlass) |
---|
| 2536 | |
---|
| 2537 | |
---|
| 2538 | def eatErrorCatcher(self): |
---|
| 2539 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2540 | endOfFirstLinePos = self.findEOL() |
---|
| 2541 | self.getDirectiveStartToken() |
---|
| 2542 | self.advance(len('errorCatcher')) |
---|
| 2543 | self.getWhiteSpace() |
---|
| 2544 | startPos = self.pos() |
---|
| 2545 | errorCatcherName = self.getIdentifier() |
---|
| 2546 | errorCatcherName = self._applyExpressionFilters( |
---|
| 2547 | errorCatcherName, 'errorcatcher', startPos=startPos) |
---|
| 2548 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2549 | self._compiler.setErrorCatcher(errorCatcherName) |
---|
| 2550 | |
---|
| 2551 | def eatCapture(self): |
---|
| 2552 | # @@TR: this could be refactored to use the code in eatSimpleIndentingDirective |
---|
| 2553 | # filtered |
---|
| 2554 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2555 | endOfFirstLinePos = self.findEOL() |
---|
| 2556 | lineCol = self.getRowCol() |
---|
| 2557 | |
---|
| 2558 | self.getDirectiveStartToken() |
---|
| 2559 | self.advance(len('capture')) |
---|
| 2560 | startPos = self.pos() |
---|
| 2561 | self.getWhiteSpace() |
---|
| 2562 | |
---|
| 2563 | expr = self.getExpression(pyTokensToBreakAt=[':']) |
---|
| 2564 | expr = self._applyExpressionFilters(expr, 'capture', startPos=startPos) |
---|
| 2565 | if self.matchColonForSingleLineShortFormDirective(): |
---|
| 2566 | self.advance() # skip over : |
---|
| 2567 | self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol) |
---|
| 2568 | self.getWhiteSpace(max=1) |
---|
| 2569 | self.parse(breakPoint=self.findEOL(gobble=False)) |
---|
| 2570 | self._compiler.endCaptureRegion() |
---|
| 2571 | else: |
---|
| 2572 | if self.peek()==':': |
---|
| 2573 | self.advance() |
---|
| 2574 | self.getWhiteSpace() |
---|
| 2575 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) |
---|
| 2576 | self.pushToOpenDirectivesStack("capture") |
---|
| 2577 | self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol) |
---|
| 2578 | |
---|
| 2579 | |
---|
| 2580 | def eatIf(self): |
---|
| 2581 | # filtered |
---|
| 2582 | isLineClearToStartToken = self.isLineClearToStartToken() |
---|
| 2583 | endOfFirstLine = self.findEOL() |
---|
| 2584 | lineCol = self.getRowCol() |
---|
| 2585 | self.getDirectiveStartToken() |
---|
| 2586 | startPos = self.pos() |
---|
| 2587 | |
---|
| 2588 | expressionParts = self.getExpressionParts(pyTokensToBreakAt=[':']) |
---|
| 2589 | expr = ''.join(expressionParts).strip() |
---|
| 2590 | expr = self._applyExpressionFilters(expr, 'if', startPos=startPos) |
---|
| 2591 | |
---|
| 2592 | isTernaryExpr = ('then' in expressionParts and 'else' in expressionParts) |
---|
| 2593 | if isTernaryExpr: |
---|
| 2594 | conditionExpr = [] |
---|
| 2595 | trueExpr = [] |
---|
| 2596 | falseExpr = [] |
---|
| 2597 | currentExpr = conditionExpr |
---|
| 2598 | for part in expressionParts: |
---|
| 2599 | if part.strip()=='then': |
---|
| 2600 | currentExpr = trueExpr |
---|
| 2601 | elif part.strip()=='else': |
---|
| 2602 | currentExpr = falseExpr |
---|
| 2603 | else: |
---|
| 2604 | currentExpr.append(part) |
---|
| 2605 | |
---|
| 2606 | conditionExpr = ''.join(conditionExpr) |
---|
| 2607 | trueExpr = ''.join(trueExpr) |
---|
| 2608 | falseExpr = ''.join(falseExpr) |
---|
| 2609 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 2610 | self._compiler.addTernaryExpr(conditionExpr, trueExpr, falseExpr, lineCol=lineCol) |
---|
| 2611 | elif self.matchColonForSingleLineShortFormDirective(): |
---|
| 2612 | self.advance() # skip over : |
---|
| 2613 | self._compiler.addIf(expr, lineCol=lineCol) |
---|
| 2614 | self.getWhiteSpace(max=1) |
---|
| 2615 | self.parse(breakPoint=self.findEOL(gobble=True)) |
---|
| 2616 | self._compiler.commitStrConst() |
---|
| 2617 | self._compiler.dedent() |
---|
| 2618 | else: |
---|
| 2619 | if self.peek()==':': |
---|
| 2620 | self.advance() |
---|
| 2621 | self.getWhiteSpace() |
---|
| 2622 | self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) |
---|
| 2623 | self.pushToOpenDirectivesStack('if') |
---|
| 2624 | self._compiler.addIf(expr, lineCol=lineCol) |
---|
| 2625 | |
---|
| 2626 | ## end directive handlers |
---|
| 2627 | def handleEndDef(self): |
---|
| 2628 | isNestedDef = (self.setting('allowNestedDefScopes') |
---|
| 2629 | and [name for name in self._openDirectivesStack if name=='def']) |
---|
| 2630 | if not isNestedDef: |
---|
| 2631 | self._compiler.closeDef() |
---|
| 2632 | else: |
---|
| 2633 | # @@TR: temporary hack of useSearchList |
---|
| 2634 | self.setSetting('useSearchList', self._useSearchList_orig) |
---|
| 2635 | self._compiler.commitStrConst() |
---|
| 2636 | self._compiler.dedent() |
---|
| 2637 | ### |
---|
| 2638 | |
---|
| 2639 | def pushToOpenDirectivesStack(self, directiveName): |
---|
| 2640 | assert directiveName in self._closeableDirectives |
---|
| 2641 | self._openDirectivesStack.append(directiveName) |
---|
| 2642 | |
---|
| 2643 | def popFromOpenDirectivesStack(self, directiveName): |
---|
| 2644 | if not self._openDirectivesStack: |
---|
| 2645 | raise ParseError(self, msg="#end found, but nothing to end") |
---|
| 2646 | |
---|
| 2647 | if self._openDirectivesStack[-1] == directiveName: |
---|
| 2648 | del self._openDirectivesStack[-1] |
---|
| 2649 | else: |
---|
| 2650 | raise ParseError(self, msg="#end %s found, expected #end %s" %( |
---|
| 2651 | directiveName, self._openDirectivesStack[-1])) |
---|
| 2652 | |
---|
| 2653 | def assertEmptyOpenDirectivesStack(self): |
---|
| 2654 | if self._openDirectivesStack: |
---|
| 2655 | errorMsg = ( |
---|
| 2656 | "Some #directives are missing their corresponding #end ___ tag: %s" %( |
---|
| 2657 | ', '.join(self._openDirectivesStack))) |
---|
| 2658 | raise ParseError(self, msg=errorMsg) |
---|
| 2659 | |
---|
| 2660 | ################################################## |
---|
| 2661 | ## Make an alias to export |
---|
| 2662 | Parser = _HighLevelParser |
---|