Package lepl :: Package matchers :: Module support
[hide private]
[frames] | no frames]

Source Code for Module lepl.matchers.support

  1   
  2  # The contents of this file are subject to the Mozilla Public License 
  3  # (MPL) Version 1.1 (the "License"); you may not use this file except 
  4  # in compliance with the License. You may obtain a copy of the License 
  5  # at http://www.mozilla.org/MPL/ 
  6  # 
  7  # Software distributed under the License is distributed on an "AS IS" 
  8  # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 
  9  # the License for the specific language governing rights and 
 10  # limitations under the License. 
 11  # 
 12  # The Original Code is LEPL (http://www.acooke.org/lepl) 
 13  # The Initial Developer of the Original Code is Andrew Cooke. 
 14  # Portions created by the Initial Developer are Copyright (C) 2009-2010 
 15  # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 
 16  # 
 17  # Alternatively, the contents of this file may be used under the terms 
 18  # of the LGPL license (the GNU Lesser General Public License, 
 19  # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 
 20  # of the LGPL License are applicable instead of those above. 
 21  # 
 22  # If you wish to allow use of your version of this file only under the 
 23  # terms of the LGPL License and not to allow others to use your version 
 24  # of this file under the MPL, indicate your decision by deleting the 
 25  # provisions above and replace them with the notice and other provisions 
 26  # required by the LGPL License.  If you do not delete the provisions 
 27  # above, a recipient may use your version of this file under either the 
 28  # MPL or the LGPL License. 
 29   
 30  ''' 
 31  Support classes for matchers. 
 32  ''' 
 33   
 34  from abc import ABCMeta 
 35  from inspect import getargspec 
 36   
 37  from lepl.core.config import ParserMixin 
 38  from lepl.core.parser import GeneratorWrapper, tagged 
 39  from lepl.support.graph import ArgAsAttributeMixin, PostorderWalkerMixin, \ 
 40      GraphStr 
 41  from lepl.matchers.matcher import Matcher, FactoryMatcher, add_child, is_child 
 42  from lepl.matchers.operators import OperatorMixin, OPERATORS, \ 
 43      OperatorNamespace 
 44  from lepl.support.lib import LogMixin, basestring, fmt, document, identity 
 45   
 46  # pylint: disable-msg=C0103,W0212 
 47  # (consistent interfaces) 
 48  # pylint: disable-msg=E1101 
 49  # (_args create attributes) 
 50  # pylint: disable-msg=R0901, R0904, W0142 
 51  # lepl conventions 
 52   
 53   
 54  # pylint: disable-msg=W0105, C0103, R0903, W0212 
 55  # Python 2.6 
 56  #class NoMemo(metaclass=NoMemo): 
 57  NoMemo = ABCMeta('NoMemo', (object, ), {}) 
 58  ''' 
 59  ABC used to indicate that memoisation should not be applied.   
 60  ''' 
61 62 63 -class BaseMatcher(ArgAsAttributeMixin, PostorderWalkerMixin, 64 LogMixin, Matcher):
65 ''' 66 A base class that provides support to all matchers. 67 ''' 68
69 - def __repr__(self):
70 return self._indented_repr(0, set())
71
72 - def _fmt_repr(self, indent, value, visited, key=None):
73 prefix = (' ' * indent) + (key + '=' if key else '') 74 if is_child(value, Matcher, fail=False): 75 if value in visited: 76 return prefix + '[' + value._small_str + ']' 77 else: 78 return value._indented_repr(indent, visited, key) 79 else: 80 return prefix + repr(value)
81
82 - def _format_repr(self, indent, key, contents):
83 return fmt('{0}{1}{2}({3}{4})', 84 ' ' * indent, 85 key + '=' if key else '', 86 self._small_str, 87 '' if self._fmt_compact else '\n', 88 ',\n'.join(contents))
89
90 - def _indented_repr(self, indent0, visited, key=None):
91 visited = set(visited) # copy so only block parents 92 visited.add(self) 93 (args, kargs) = self._constructor_args() 94 indent1 = 0 if self._fmt_compact else indent0 + 1 95 contents = [self._fmt_repr(indent1, arg, visited) for arg in args] + \ 96 [self._fmt_repr(indent1, kargs[key], visited, key) for key in kargs] 97 return self._format_repr(indent0, key, contents)
98 99 @property
100 - def _fmt_compact(self):
101 (args, kargs) = self._constructor_args() 102 if len(args) + len(kargs) > 1: 103 return False 104 for arg in args: 105 try: 106 if not arg._fmt_compact: 107 return False 108 except AttributeError: 109 pass 110 for arg in kargs: 111 try: 112 if not arg._fmt_compact: 113 return False 114 except AttributeError: 115 pass 116 return True
117
118 - def _fmt_str(self, value, key=None):
119 return (key + '=' if key else '') + \ 120 value._small_str if isinstance(value, Matcher) else repr(value)
121
122 - def __str__(self):
123 (args, kargs) = self._constructor_args() 124 contents = [self._fmt_str(arg) for arg in args] + \ 125 [self._fmt_str(kargs[key], key) for key in kargs] 126 return fmt('{0}({1})', self._small_str, 127 ', '.join(contents))
128 129 @property
130 - def kargs(self):
131 (_, kargs) = self._constructor_args() 132 return kargs
133
134 - def tree(self):
135 ''' 136 An ASCII tree for display. 137 ''' 138 visitor = TreeStr() 139 return self.postorder(visitor, Matcher)
140
141 - def tree_repr(self):
142 ''' 143 The text to display via `tree()` 144 ''' 145 return self.__class__.__name__
146
147 - def clone(self):
148 ''' 149 Clone this matcher (and associated graph). This is currently only used 150 when we want to test different configurations. 151 ''' 152 from lepl.core.rewriters import clone_matcher 153 return clone_matcher(self)
154
155 156 -class TreeStr(GraphStr):
157
158 - def node(self, node):
159 ''' 160 Store the class name. 161 ''' 162 try: 163 self._type = node.tree_repr() 164 except: 165 super(TreeStr, self).node(node)
166
167 168 -class OperatorMatcher(OperatorMixin, ParserMixin, BaseMatcher):
169 ''' 170 A base class that provides support to all matchers with operators. 171 ''' 172
173 - def __init__(self, name=OPERATORS, namespace=OperatorNamespace):
174 super(OperatorMatcher, self).__init__(name=name, namespace=namespace)
175
176 177 -class Transformable(OperatorMatcher):
178 ''' 179 All subclasses must expose and apply wrapper, and implement compose. 180 181 This allows `Transform` instances to be merged directly. 182 ''' 183
184 - def __init__(self, function=None):
185 from lepl.matchers.transform import TransformationWrapper 186 super(Transformable, self).__init__() 187 if not isinstance(function, TransformationWrapper): 188 function = TransformationWrapper(function) 189 self.wrapper = function
190
191 - def compose(self, wrapper):
192 ''' 193 Combine with a transformation wrapper, returning a new instance. 194 195 We must return a new instance because the same Transformable may 196 occur more than once in a graph and we don't want to include the 197 transformation in other cases. 198 ''' 199 raise NotImplementedError()
200
201 - def _format_repr(self, indent, key, contents):
202 return fmt('{0}{1}{2}({3}{4})', 203 ' ' * indent, 204 key + '=' if key else '', 205 self.tree_repr(), 206 '' if self._fmt_compact else '\n', 207 ',\n'.join(contents))
208
209 - def tree_repr(self):
210 return fmt('{0}:{1}', 211 self._small_str, 212 self.wrapper)
213
214 215 -class BaseFactoryMatcher(FactoryMatcher):
216 ''' 217 This must be used as a mixin with something that inherits from 218 ArgsAsAttribute (ie the usual matcher classes). 219 ''' 220
221 - def __init__(self, *args, **kargs):
222 super(FactoryMatcher, self).__init__() 223 self.__args = args 224 self.__kargs = kargs 225 self.__factory = None 226 self.__small_str = None 227 self.__cached_matcher = None
228
229 - def __args_as_attributes(self):
230 ''' 231 Validate the arguments passed to the constructor against the spec for 232 the factory (necessary because we use *args and so the user doesn't 233 get the feedback they will expect if they make a mistake). As a side 234 effect we also associated arguments with names and expand defaults 235 so that attributes are more predictable. 236 ''' 237 try: 238 # function wrapper, so we have two levels, and we must construct 239 # a new, empty function (ie this is a fake function that helps 240 # us use the code below, even though there's no arguments because 241 # factory is a dummy generated in make_factory below) 242 def empty(): return 243 document(empty, self.factory.factory) 244 spec = getargspec(empty) 245 except: 246 spec = getargspec(self.factory) 247 names = list(spec.args) 248 defaults = dict(zip(names[::-1], spec.defaults[::-1] if spec.defaults else [])) 249 for name in names: 250 if name in self.__kargs: 251 self._karg(**{name: self.__kargs[name]}) 252 del self.__kargs[name] 253 elif self.__args: 254 self._arg(**{name: self.__args[0]}) 255 self.__args = self.__args[1:] 256 elif name in defaults: 257 self._karg(**{name: defaults[name]}) 258 else: 259 raise TypeError(fmt("No value for argument '{0}' in " 260 "{1}(...)", 261 name, self._small_str)) 262 if self.__args: 263 if spec.varargs: 264 self._args(**{spec.varargs: self.__args}) 265 else: 266 raise TypeError(fmt("No parameter matches the argument " 267 "{0!r} in {1}(...)", 268 self.__args[0], self._small_str)) 269 if self.__kargs: 270 if spec.keywords: 271 self.__kargs(**{spec.keywords: self.__kargs}) 272 else: 273 name = list(self.__kargs.keys())[0] 274 value = self.__kargs[name] 275 raise TypeError(fmt("No parameter matches the argument " 276 "{0}={1!r} in {2}(...)", 277 name, value, self._small_str))
278 279 @property
280 - def factory(self):
281 return self.__factory
282 283 @factory.setter
284 - def factory(self, factory):
285 if not self.__factory: 286 assert factory 287 self.__factory = factory 288 self._small_str = self.__small_str if self.__small_str \ 289 else factory.__name__ 290 self.__args_as_attributes()
291
292 - def tree_repr(self):
293 return fmt('{0}<{1}>', 294 self.__class__.__name__, 295 self._small_str)
296 297 @property
298 - def _cached_matcher(self):
299 if not self.__cached_matcher: 300 (args, kargs) = self._constructor_args() 301 self.__cached_matcher = self.factory(*args, **kargs) 302 return self.__cached_matcher
303
304 305 -class TransformableWrapper(BaseFactoryMatcher, Transformable):
306 ''' 307 A common support class for various wrappers. 308 ''' 309
310 - def compose(self, wrapper):
311 (args, kargs) = self._constructor_args() 312 copy = type(self)(*args, **kargs) 313 copy.factory = self.factory 314 copy.wrapper = self.wrapper.compose(wrapper) 315 return copy
316
317 - def tree_repr(self):
318 return fmt('{0}<{1}:{2}>', 319 self.__class__.__name__, 320 self._small_str, 321 self.wrapper)
322
323 324 -class TransformableTrampolineWrapper(TransformableWrapper):
325 ''' 326 A wrapper for source of generators that evaluate other matchers via 327 the trampoline (ie for generators that evaluate matchers via yield). 328 329 Typically only used for advanced matchers. 330 331 NOTE: This is no longer used; the `TrampolineWrapper` is used instead. 332 ''' 333 334 @tagged
335 - def _match(self, stream_in):
336 from lepl.matchers.transform import raise_ 337 function = self.wrapper.function 338 generator = GeneratorWrapper(self._cached_matcher(self, stream_in), 339 self, stream_in) 340 stream_in = None 341 while True: 342 try: 343 value = yield generator 344 yield function(stream_in, lambda: value) \ 345 if function else value 346 except StopIteration as e: 347 if function: 348 yield function(stream_in, lambda: raise_(StopIteration)) 349 else: 350 raise e
351
352 353 -class TrampolineWrapper(BaseFactoryMatcher, OperatorMatcher):
354 ''' 355 A wrapper for source of generators that evaluate other matchers via 356 the trampoline (ie for generators that evaluate matchers via yield). 357 358 Typically only used for advanced matchers. 359 360 This improves on `TransformableTrampolineWrapepr` because it does not add 361 an extra layer of trampolining when no transform is needed. When a 362 transform is needed then a `Transform` matcher adds an extra layer anyway 363 (the motivation for the original code was to allow the two to merge, but 364 there is no point if the unmerged version is just as slow). 365 ''' 366 367 @tagged
368 - def _match(self, stream_in):
369 return self._cached_matcher(self, stream_in)
370
371 372 -class NoTrampoline(object):
373 ''' 374 A matcher that does not require the trampoline. 375 ''' 376 377 #@abstractmethod
378 - def _untagged_match(self, stream):
379 ''' 380 This should work like `_match()`, but without any tagged wrapper. 381 382 It would be nice if both could be generated dynamically, but 383 cut + paste appears to be faster, and this is an optimisation. 384 '''
385
386 387 -class NoTrampolineWrapper(NoTrampoline, TransformableWrapper):
388 ''' 389 A wrapper for source of generators that do not evaluate other matchers via 390 the trampoline. 391 392 Subclasses can be used without trampolining via `_untagged_match`. 393 ''' 394 395 #@abstractmethod
396 - def _untagged_match(self, stream):
397 ''' 398 This should work like `_match()`, but without any tagged wrapper. 399 400 It would be nice if both could be generated dynamically, but 401 cut + paste appears to be faster, and this is an optimisation. 402 '''
403
404 405 -class SequenceWrapper(NoTrampolineWrapper):
406 ''' 407 A wrapper for simple generator factories, where the final matcher is a 408 function that yields a series of matches without evaluating other matchers 409 via the trampoline. 410 ''' 411 412 @tagged
413 - def _match(self, stream_in):
414 from lepl.matchers.transform import raise_ 415 function = self.wrapper.function 416 for results in self._cached_matcher(self, stream_in): 417 yield function(stream_in, lambda: results) if function else results 418 while function: 419 yield function(stream_in, lambda: raise_(StopIteration))
420
421 - def _untagged_match(self, stream_in):
422 from lepl.matchers.transform import raise_ 423 function = self.wrapper.function 424 for results in self._cached_matcher(self, stream_in): 425 yield function(stream_in, lambda: results) if function else results 426 while function: 427 yield function(stream_in, lambda: raise_(StopIteration))
428
429 430 -class FunctionWrapper(NoTrampolineWrapper):
431 ''' 432 A wrapper for simple function factories, where the final matcher is a 433 function that returns a single match or None. 434 ''' 435 436 @tagged
437 - def _match(self, stream_in):
438 from lepl.matchers.transform import raise_ 439 function = self.wrapper.function 440 results = self._cached_matcher(self, stream_in) 441 if results is not None: 442 yield function(stream_in, lambda: results) \ 443 if function else results 444 while function: 445 yield function(stream_in, lambda: raise_(StopIteration))
446
447 - def _untagged_match(self, stream_in):
448 from lepl.matchers.transform import raise_ 449 function = self.wrapper.function 450 results = self._cached_matcher(self, stream_in) 451 if results is not None: 452 yield function(stream_in, lambda: results) \ 453 if function else results 454 while function: 455 yield function(stream_in, lambda: raise_(StopIteration))
456
457 458 -def check_matcher(matcher):
459 ''' 460 Check that the signature takes support + stream. 461 ''' 462 check_args(matcher) 463 spec = getargspec(matcher) 464 if len(spec.args) != 2: 465 raise TypeError(fmt( 466 '''The function {0} cannot be used as a matcher because it does not have 467 exactly two parameters. 468 469 A typical definition will look like: 470 471 def {0}(support, stream): 472 ... 473 474 where 'support' is a BaseMatcher instance (support for logging, etc) and 475 'stream' is a SimpleStream instance (which supports access via stream[i] 476 and truncation via stream[i:]).''', matcher.__name__))
477
478 479 -def check_args(func):
480 ''' 481 Check that the factory doesn't use any of those modern haifalutin 482 extensions... 483 ''' 484 try: 485 getargspec(func) 486 except Exception as e: 487 raise TypeError(fmt( 488 '''The function {0} uses Python 3 style parameters (keyword only, etc). 489 These are not supported by LEPL factory wrappers currently. If you really 490 need this functionality, subclass BaseMatcher.''', func.__name__))
491
492 493 -def check_modifiers(func, modifiers):
494 ''' 495 Check that any modifiers match the function declaration. 496 ''' 497 argspec = getargspec(func) 498 for name in modifiers: 499 if name not in argspec.args: 500 raise TypeError( 501 fmt("A modifier was specified for argument {'0}' " 502 "in {1}(), but is not declared.", name, func.__name__))
503
504 505 -def apply_modifiers(func, args, kargs, modifiers, margs, mkargs):
506 ''' 507 Modify values in args and kargs. 508 ''' 509 spec = getargspec(func) 510 names = list(spec.args) 511 defaults = dict(zip(names[::-1], spec.defaults[::-1] if spec.defaults else [])) 512 newargs = [] 513 newkargs = {} 514 for name in names: 515 if name in kargs: 516 value = kargs[name] 517 if name in modifiers: 518 value = modifiers[name](value) 519 newkargs[name] = value 520 del kargs[name] 521 elif args: 522 (value, args) = (args[0], args[1:]) 523 if name in modifiers: 524 value = modifiers[name](value) 525 newargs.append(value) 526 elif name in defaults: 527 value = defaults[name] 528 if name in modifiers: 529 value = modifiers[name](value) 530 newkargs[name] = value 531 else: 532 raise TypeError(fmt("No value for argument '{0}' in " 533 "{1}(...)", name, func.__name__)) 534 # copy across varags 535 if spec.varargs: 536 newargs.extend(map(margs, args)) 537 elif args: 538 raise TypeError(fmt("Unexpected argument {0!r} for {1}(...)", 539 args[0], func.__name__)) 540 if spec.keywords: 541 for name in kargs: 542 newkargs[name] = mkargs(kargs[name]) 543 elif kargs: 544 for name in kargs: 545 raise TypeError(fmt("Unexpected argument {0}={1!r} for {2}(...)", 546 name, kargs[name], func.__name__)) 547 return (newargs, newkargs)
548
549 550 -def make_wrapper_factory(wrapper, factory, modifiers, 551 margs=identity, mkargs=identity, memo=True):
552 ''' 553 A helper function that assembles a matcher from a wrapper class and 554 a factory function that contains the logic. 555 556 `wrapper` is the class that will be constructed, and which will contain 557 the functionality defined in `factory`. 558 559 `modifiers` is a map of functions applied to arguments of the given name. 560 Similarly, `margs` and `mkargs` apply to varargs. 561 562 `memo` should be set to False for matchers that do not want memoisation. 563 ''' 564 check_args(factory) 565 check_modifiers(factory, modifiers) 566 def wrapper_factory(*args, **kargs): 567 (args, kargs) = \ 568 apply_modifiers(factory, args, kargs, modifiers, margs, mkargs) 569 made = wrapper(*args, **kargs) 570 made.factory = factory 571 return made
572 wrapper_factory.factory = factory 573 add_child(Matcher, wrapper_factory) 574 if not memo: 575 add_child(NoMemo, wrapper_factory) 576 # module used in auto-linking for docs 577 wrapper_factory.__module__ = factory.__module__ 578 return wrapper_factory 579
580 581 -def make_factory(maker, matcher):
582 ''' 583 A helper function that assembles a matcher from a wrapper class and 584 a function that contains the logic. 585 586 This works by generating a dummy factory and delegating to 587 `make_wrapper_factory`. 588 ''' 589 def factory(*args, **kargs): 590 if args or kargs: 591 raise TypeError(fmt('{0}() takes no arguments', 592 matcher.__name__)) 593 return matcher
594 document(factory, matcher) 595 factory.factory = matcher 596 return maker(factory) 597
598 599 -def keep_gate(gatekeeper, name):
600 if gatekeeper: 601 raise ValueError( 602 '%s must be used as a function:' 603 '\n @%s()' 604 '\n def MyMatcherFactory(...):' 605 '\n ....' % (name, name))
606
607 608 -def trampoline_matcher_factory(gatekeeper_=None, 609 args_=identity, kargs_=identity, **kargs):
610 ''' 611 Decorator that allows matchers to be defined using a nested pair 612 of functions. The outer function acts like a constructor; the inner 613 function implements the matcher logic. 614 615 The matcher code can evaluate sub-matchers by yielding the generator 616 created by `matcher._match()` to the trampoline. Matches should also 617 be yielded. 618 619 Other keyword arguments should match factory arguments and identify 620 functions of one argument that are applied to the arguments when the 621 matcher is created as part of a grammar (eg coercion). Similarly, 622 `args_` and `kargs_` are applied to varargs. 623 ''' 624 keep_gate(gatekeeper_, 'trampoline_matcher_factory') 625 def wrapper(factory): 626 # return make_wrapper_factory(TransformableTrampolineWrapper, 627 # factory, kargs, args_, kargs_) 628 return make_wrapper_factory(TrampolineWrapper, 629 factory, kargs, args_, kargs_)
630 return wrapper 631
632 -def trampoline_matcher(matcher):
633 ''' 634 Decorator that allows matchers to be defined using a single function 635 to implement the matcher logic. 636 637 The matcher code can evaluate sub-matchers by yielding the generator 638 created by `matcher._match()` to the trampoline. Matches should also 639 be yielded. 640 ''' 641 check_matcher(matcher) 642 return make_factory(trampoline_matcher_factory(), matcher)
643
644 -def sequence_matcher_factory(gatekeeper_=None, 645 args_=identity, kargs_=identity, **kargs):
646 ''' 647 Decorator that allows matchers to be defined using a nested pair 648 of functions. The outer function acts like a constructor; the inner 649 function implements the matcher logic. 650 651 The matcher must yield matches (multiple times if required). It 652 *cannot* evaluate sub-matchers. 653 654 `args_`, `kargs_` and named arguments define modifiers, which are applied 655 to the corresponding arguments when the matcher is first used in a grammar 656 (eg to coerce strings to `Literal` instances). 657 ''' 658 keep_gate(gatekeeper_, 'sequence_matcher_factory') 659 def wrapper(factory): 660 return make_wrapper_factory(SequenceWrapper, factory, 661 kargs, args_, kargs_)
662 return wrapper 663
664 -def sequence_matcher(matcher):
665 ''' 666 Decorator that allows matchers to be defined using a single function 667 to implement the matcher logic. 668 669 The matcher must yield matches (multiple times if required). It 670 *cannot* evaluate sub-matchers. 671 ''' 672 check_matcher(matcher) 673 return make_factory(sequence_matcher_factory(), matcher)
674
675 -def function_matcher_factory(gatekeeper_=None, 676 args_=identity, kargs_=identity, **kargs):
677 ''' 678 Decorator that allows matchers to be defined using a nested pair 679 of functions. The outer function acts like a constructor; the inner 680 function implements the matcher logic. 681 682 The matcher must return a single match. It *cannot* evaluate sub-matchers. 683 684 `args_`, `kargs_` and named arguments define modifiers, which are applied 685 to the corresponding arguments when the matcher is first used in a grammar 686 (eg to coerce strings to `Literal` instances). 687 ''' 688 keep_gate(gatekeeper_, 'function_matcher_factory') 689 def wrapper(factory): 690 return make_wrapper_factory(FunctionWrapper, factory, 691 kargs, args_, kargs_)
692 return wrapper 693
694 -def function_matcher(matcher):
695 ''' 696 Decorator that allows matchers to be defined using a single function 697 to implement the matcher logic. 698 699 The matcher must return a single match. It *cannot* evaluate sub-matchers. 700 ''' 701 check_matcher(matcher) 702 return make_factory(function_matcher_factory(), matcher)
703
704 705 -def coerce_(arg, function=None):
706 ''' 707 Many arguments can take a string which is implicitly converted (via this 708 function) to a literal (or similar). 709 ''' 710 if function is None: 711 from lepl.matchers.core import Literal 712 function = Literal 713 return function(arg) if isinstance(arg, basestring) else arg
714
715 716 -def to(function):
717 ''' 718 Generate a coercion for later application. 719 ''' 720 return lambda matcher: coerce_(matcher, function)
721