Package lepl :: Package lexer :: Package lines :: Module matchers
[hide private]
[frames] | no frames]

Source Code for Module lepl.lexer.lines.matchers

  1   
  2  # The contents of this file are subject to the Mozilla Public License 
  3  # (MPL) Version 1.1 (the "License"); you may not use this file except 
  4  # in compliance with the License. You may obtain a copy of the License 
  5  # at http://www.mozilla.org/MPL/ 
  6  # 
  7  # Software distributed under the License is distributed on an "AS IS" 
  8  # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 
  9  # the License for the specific language governing rights and 
 10  # limitations under the License. 
 11  # 
 12  # The Original Code is LEPL (http://www.acooke.org/lepl) 
 13  # The Initial Developer of the Original Code is Andrew Cooke. 
 14  # Portions created by the Initial Developer are Copyright (C) 2009-2010 
 15  # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 
 16  # 
 17  # Alternatively, the contents of this file may be used under the terms 
 18  # of the LGPL license (the GNU Lesser General Public License, 
 19  # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 
 20  # of the LGPL License are applicable instead of those above. 
 21  # 
 22  # If you wish to allow use of your version of this file only under the 
 23  # terms of the LGPL License and not to allow others to use your version 
 24  # of this file under the MPL, indicate your decision by deleting the 
 25  # provisions above and replace them with the notice and other provisions 
 26  # required by the LGPL License.  If you do not delete the provisions 
 27  # above, a recipient may use your version of this file under either the 
 28  # MPL or the LGPL License. 
 29   
 30  from lepl.lexer.matchers import Token, RestrictTokensBy, EmptyToken 
 31  from lepl.lexer.lines.lexer import START, END 
 32  from lepl.lexer.lines.monitor import BlockMonitor 
 33  from lepl.matchers.support import coerce_, OperatorMatcher, NoMemo 
 34  from lepl.core.parser import tagged 
 35  from lepl.support.lib import fmt 
 36  from lepl.matchers.combine import And 
 37  from lepl.stream.core import s_key, s_next, s_line 
 38   
 39   
 40  NO_BLOCKS = object() 
 41  ''' 
 42  Magic initial value for block_offset to disable indentation checks. 
 43  ''' 
44 45 46 -class LineStart(Token):
47
48 - def __init__(self, indent=True, regexp=None, content=None, id_=None, 49 alphabet=None, complete=True, compiled=False):
50 ''' 51 Arguments used only to support cloning. 52 ''' 53 super(LineStart, self).__init__(regexp=None, content=None, id_=START, 54 alphabet=None, complete=True, 55 compiled=compiled) 56 self._karg(indent=indent) 57 self.monitor_class = BlockMonitor 58 self._current_indent = NO_BLOCKS
59
60 - def on_push(self, monitor):
61 ''' 62 Read the global indentation level. 63 ''' 64 if self.indent: 65 self._current_indent = monitor.indent
66
67 - def on_pop(self, monitor):
68 ''' 69 Unused 70 '''
71 72 @tagged
73 - def _match(self, stream_in):
74 ''' 75 Check that we match the current level 76 ''' 77 try: 78 generator = super(LineStart, self)._match(stream_in) 79 while True: 80 (indent, stream) = yield generator 81 self._debug(fmt('SOL {0!r}', indent)) 82 if indent and indent[0] and indent[0][-1] == '\n': 83 indent[0] = indent[0][:-1] 84 # if we're not doing indents, this is empty 85 if not self.indent: 86 yield ([], stream) 87 # if we are doing indents, we need a match or NO_BLOCKS 88 elif self._current_indent == NO_BLOCKS or \ 89 len(indent[0]) == self._current_indent: 90 yield (indent, stream) 91 else: 92 self._debug( 93 fmt('Incorrect indent ({0:d} != len({1!r}), {2:d})', 94 self._current_indent, indent[0], 95 len(indent[0]))) 96 except StopIteration: 97 pass
98
99 100 -class LineEnd(EmptyToken):
101
102 - def __init__(self, regexp=None, content=None, id_=None, alphabet=None, 103 complete=True, compiled=False):
104 ''' 105 Arguments used only to support cloning. 106 ''' 107 super(LineEnd, self).__init__(regexp=None, content=None, id_=END, 108 alphabet=None, complete=True, 109 compiled=compiled)
110
111 112 -def Line(matcher, indent=True):
113 ''' 114 Match the matcher within a line. 115 ''' 116 return ~LineStart(indent=indent) & matcher & ~LineEnd()
117
118 119 -def ContinuedLineFactory(matcher):
120 ''' 121 Create a replacement for ``Line()`` that can match multiple lines if they 122 end in the given character/matcher. 123 ''' 124 matcher = coerce_(matcher, lambda regexp: Token(regexp)) 125 restricted = RestrictTokensBy(matcher, LineEnd(), LineStart()) 126 def factory(matcher, indent=True): 127 return restricted(Line(matcher, indent=indent))
128 return factory 129
130 131 -def Extend(matcher):
132 ''' 133 Apply the given matcher to a token stream that ignores line endings and 134 starts (so it matches over multiple lines). 135 ''' 136 start = LineStart() 137 end = LineEnd() 138 return RestrictTokensBy(end, start)(matcher)
139 140 141 # pylint: disable-msg=W0105 142 # epydoc convention 143 DEFAULT_TABSIZE = 8 144 ''' 145 The default number of spaces for a tab. 146 '''
147 148 -def constant_indent(n_spaces):
149 ''' 150 Construct a simple policy for `Block` that increments the indent 151 by some fixed number of spaces. 152 ''' 153 def policy(current, _indent): 154 ''' 155 Increment current by n_spaces 156 ''' 157 return current + n_spaces
158 return policy 159
160 161 -def explicit(_current, indent):
162 ''' 163 Another simple policy that matches whatever indent is used. 164 ''' 165 return len(indent)
166
167 168 -def to_right(current, indent):
169 ''' 170 This allows new blocks to be used without any introduction (eg no colon 171 on the preceding line). See the "closed_bug" test for more details. 172 ''' 173 new = len(indent) 174 if new <= current: 175 raise StopIteration 176 return new
177 178 179 DEFAULT_POLICY = constant_indent(DEFAULT_TABSIZE) 180 ''' 181 By default, expect an indent equivalent to a tab. 182 '''
183 184 # pylint: disable-msg=E1101, W0212, R0901, R0904 185 # pylint conventions 186 -class Block(OperatorMatcher, NoMemo):
187 ''' 188 Set a new indent level for the enclosed matchers (typically `BLine` and 189 `Block` instances). 190 191 In the simplest case, this might increment the global indent by 4, say. 192 In a more complex case it might look at the current token, expecting an 193 `Indent`, and set the global indent at that amount if it is larger 194 than the current value. 195 196 A block will always match an `Indent`, but will not consume it 197 (it will remain in the stream after the block has finished). 198 199 The usual memoization of left recursive calls will not detect problems 200 with nested blocks (because the indentation changes), so instead we 201 track and block nested calls manually. 202 ''' 203 204 POLICY = 'policy' 205 206 # Python 2.6 does not support this syntax 207 # def __init__(self, *lines, policy=None, indent=None):
208 - def __init__(self, *lines, **kargs):
209 ''' 210 Lines are invoked in sequence (like `And()`). 211 212 The policy is passed the current level and the indent and must 213 return a new level. Typically it is set globally by rewriting with 214 a default in the configuration. If it is given as an integer then 215 `constant_indent` is used to create a policy from that. 216 217 indent is the matcher used to match indents, and is exposed for 218 rewriting/extension (in other words, ignore it). 219 ''' 220 super(Block, self).__init__() 221 self._args(lines=lines) 222 policy = kargs.get(self.POLICY, DEFAULT_POLICY) 223 if isinstance(policy, int): 224 policy = constant_indent(policy) 225 self._karg(policy=policy) 226 self.monitor_class = BlockMonitor 227 self.__monitor = None 228 self.__streams = set()
229
230 - def on_push(self, monitor):
231 ''' 232 Store a reference to the monitor which we will update inside _match 233 ''' 234 self.__monitor = monitor
235
236 - def on_pop(self, monitor):
237 pass
238 239 @tagged
240 - def _match(self, stream_in):
241 ''' 242 Pull indent and call the policy and update the global value, 243 then evaluate the contents. 244 ''' 245 # detect a nested call 246 key = s_key(stream_in) 247 if key in self.__streams: 248 self._debug('Avoided left recursive call to Block.') 249 return 250 self.__streams.add(key) 251 try: 252 ((tokens, token_stream), _) = s_next(stream_in) 253 (indent, _) = s_line(token_stream, True) 254 if START not in tokens: 255 raise StopIteration 256 current = self.__monitor.indent 257 policy = self.policy(current, indent) 258 259 generator = And(*self.lines)._match(stream_in) 260 while True: 261 self.__monitor.push_level(policy) 262 try: 263 results = yield generator 264 finally: 265 self.__monitor.pop_level() 266 yield results 267 finally: 268 self.__streams.remove(key)
269