Source Code for Module lepl.lexer.lines.matchers

  1   
  2  # The contents of this file are subject to the Mozilla Public License 
  3  # (MPL) Version 1.1 (the "License"); you may not use this file except 
  4  # in compliance with the License. You may obtain a copy of the License 
  5  # at http://www.mozilla.org/MPL/ 
  6  # 
  7  # Software distributed under the License is distributed on an "AS IS" 
  8  # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 
  9  # the License for the specific language governing rights and 
 10  # limitations under the License. 
 11  # 
 12  # The Original Code is LEPL (http://www.acooke.org/lepl) 
 13  # The Initial Developer of the Original Code is Andrew Cooke. 
 14  # Portions created by the Initial Developer are Copyright (C) 2009-2010 
 15  # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 
 16  # 
 17  # Alternatively, the contents of this file may be used under the terms 
 18  # of the LGPL license (the GNU Lesser General Public License, 
 19  # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 
 20  # of the LGPL License are applicable instead of those above. 
 21  # 
 22  # If you wish to allow use of your version of this file only under the 
 23  # terms of the LGPL License and not to allow others to use your version 
 24  # of this file under the MPL, indicate your decision by deleting the 
 25  # provisions above and replace them with the notice and other provisions 
 26  # required by the LGPL License.  If you do not delete the provisions 
 27  # above, a recipient may use your version of this file under either the 
 28  # MPL or the LGPL License. 
 29   
 30  from lepl.lexer.matchers import Token, RestrictTokensBy, EmptyToken 
 31  from lepl.lexer.lines.lexer import START, END 
 32  from lepl.lexer.lines.monitor import BlockMonitor 
 33  from lepl.matchers.support import coerce_, OperatorMatcher, NoMemo 
 34  from lepl.core.parser import tagged 
 35  from lepl.support.lib import fmt 
 36  from lepl.matchers.combine import And 
 37  from lepl.stream.core import s_key, s_next, s_line 
 38   
 39   
 40  NO_BLOCKS = object() 
 41  ''' 
 42  Magic initial value for block_offset to disable indentation checks. 
 43  ''' 
 44   
 45   
 46 -class LineStart(Token): 
 47           
 48 -    def __init__(self, indent=True, regexp=None, content=None, id_=None,  
 49                   alphabet=None, complete=True, compiled=False): 
 50          ''' 
 51          Arguments used only to support cloning. 
 52          ''' 
 53          super(LineStart, self).__init__(regexp=None, content=None, id_=START,  
 54                                          alphabet=None, complete=True,  
 55                                          compiled=compiled) 
 56          self._karg(indent=indent) 
 57          self.monitor_class = BlockMonitor 
 58          self._current_indent = NO_BLOCKS 
 59           
 60 -    def on_push(self, monitor): 
 61          ''' 
 62          Read the global indentation level. 
 63          ''' 
 64          if self.indent: 
 65              self._current_indent = monitor.indent 
 66           
 67 -    def on_pop(self, monitor): 
 68          ''' 
 69          Unused 
 70          ''' 
 71       
 72      @tagged 
 73 -    def _match(self, stream_in): 
 74          ''' 
 75          Check that we match the current level 
 76          ''' 
 77          try: 
 78              generator = super(LineStart, self)._match(stream_in) 
 79              while True: 
 80                  (indent, stream) = yield generator 
 81                  self._debug(fmt('SOL {0!r}', indent)) 
 82                  if indent and indent[0] and indent[0][-1] == '\n':  
 83                      indent[0] = indent[0][:-1] 
 84                  # if we're not doing indents, this is empty 
 85                  if not self.indent: 
 86                      yield ([], stream) 
 87                  # if we are doing indents, we need a match or NO_BLOCKS 
 88                  elif self._current_indent == NO_BLOCKS or \ 
 89                          len(indent[0]) == self._current_indent: 
 90                      yield (indent, stream) 
 91                  else: 
 92                      self._debug( 
 93                          fmt('Incorrect indent ({0:d} != len({1!r}), {2:d})', 
 94                                 self._current_indent, indent[0],  
 95                                 len(indent[0]))) 
 96          except StopIteration: 
 97              pass 
 98   
 99   
100 -class LineEnd(EmptyToken): 
101       
102 -    def __init__(self, regexp=None, content=None, id_=None, alphabet=None, 
103                    complete=True, compiled=False): 
104          ''' 
105          Arguments used only to support cloning. 
106          ''' 
107          super(LineEnd, self).__init__(regexp=None, content=None, id_=END,  
108                                        alphabet=None, complete=True,  
109                                        compiled=compiled) 
110   
111   
112 -def Line(matcher, indent=True): 
113      ''' 
114      Match the matcher within a line. 
115      ''' 
116      return ~LineStart(indent=indent) & matcher & ~LineEnd() 
117   
118   
119 -def ContinuedLineFactory(matcher): 
120      ''' 
121      Create a replacement for ``Line()`` that can match multiple lines if they 
122      end in the given character/matcher. 
123      ''' 
124      matcher = coerce_(matcher, lambda regexp: Token(regexp)) 
125      restricted = RestrictTokensBy(matcher, LineEnd(), LineStart()) 
126      def factory(matcher, indent=True): 
127          return restricted(Line(matcher, indent=indent)) 
128      return factory 
129   
130   
131 -def Extend(matcher): 
132      ''' 
133      Apply the given matcher to a token stream that ignores line endings and 
134      starts (so it matches over multiple lines). 
135      ''' 
136      start = LineStart() 
137      end = LineEnd() 
138      return RestrictTokensBy(end, start)(matcher) 
139   
140   
141  # pylint: disable-msg=W0105 
142  # epydoc convention 
143  DEFAULT_TABSIZE = 8 
144  ''' 
145  The default number of spaces for a tab. 
146  ''' 
147   
148 -def constant_indent(n_spaces): 
149      ''' 
150      Construct a simple policy for `Block` that increments the indent 
151      by some fixed number of spaces. 
152      ''' 
153      def policy(current, _indent): 
154          ''' 
155          Increment current by n_spaces 
156          ''' 
157          return current + n_spaces 
158      return policy 
159   
160   
161 -def explicit(_current, indent): 
162      ''' 
163      Another simple policy that matches whatever indent is used. 
164      ''' 
165      return len(indent) 
166   
167   
168 -def to_right(current, indent): 
169      ''' 
170      This allows new blocks to be used without any introduction (eg no colon 
171      on the preceding line).  See the "closed_bug" test for more details. 
172      ''' 
173      new = len(indent) 
174      if new <= current: 
175          raise StopIteration 
176      return new 
177   
178   
179  DEFAULT_POLICY = constant_indent(DEFAULT_TABSIZE) 
180  ''' 
181  By default, expect an indent equivalent to a tab. 
182  ''' 
183   
184  # pylint: disable-msg=E1101, W0212, R0901, R0904 
185  # pylint conventions 
186 -class Block(OperatorMatcher, NoMemo): 
187      ''' 
188      Set a new indent level for the enclosed matchers (typically `BLine` and 
189      `Block` instances). 
190       
191      In the simplest case, this might increment the global indent by 4, say. 
192      In a more complex case it might look at the current token, expecting an 
193      `Indent`, and set the global indent at that amount if it is larger 
194      than the current value. 
195       
196      A block will always match an `Indent`, but will not consume it 
197      (it will remain in the stream after the block has finished). 
198       
199      The usual memoization of left recursive calls will not detect problems 
200      with nested blocks (because the indentation changes), so instead we 
201      track and block nested calls manually. 
202      ''' 
203       
204      POLICY = 'policy' 
205       
206  # Python 2.6 does not support this syntax 
207  #    def __init__(self, *lines, policy=None, indent=None): 
208 -    def __init__(self, *lines, **kargs): 
209          ''' 
210          Lines are invoked in sequence (like `And()`). 
211           
212          The policy is passed the current level and the indent and must  
213          return a new level.  Typically it is set globally by rewriting with 
214          a default in the configuration.  If it is given as an integer then 
215          `constant_indent` is used to create a policy from that. 
216           
217          indent is the matcher used to match indents, and is exposed for  
218          rewriting/extension (in other words, ignore it). 
219          ''' 
220          super(Block, self).__init__() 
221          self._args(lines=lines) 
222          policy = kargs.get(self.POLICY, DEFAULT_POLICY) 
223          if isinstance(policy, int): 
224              policy = constant_indent(policy) 
225          self._karg(policy=policy) 
226          self.monitor_class = BlockMonitor 
227          self.__monitor = None 
228          self.__streams = set() 
229           
230 -    def on_push(self, monitor): 
231          ''' 
232          Store a reference to the monitor which we will update inside _match 
233          ''' 
234          self.__monitor = monitor 
235           
236 -    def on_pop(self, monitor): 
237          pass 
238           
239      @tagged 
240 -    def _match(self, stream_in): 
241          ''' 
242          Pull indent and call the policy and update the global value,  
243          then evaluate the contents. 
244          ''' 
245          # detect a nested call 
246          key = s_key(stream_in) 
247          if key in self.__streams: 
248              self._debug('Avoided left recursive call to Block.') 
249              return 
250          self.__streams.add(key) 
251          try: 
252              ((tokens, token_stream), _) = s_next(stream_in) 
253              (indent, _) = s_line(token_stream, True) 
254              if START not in tokens: 
255                  raise StopIteration 
256              current = self.__monitor.indent 
257              policy = self.policy(current, indent) 
258               
259              generator = And(*self.lines)._match(stream_in) 
260              while True: 
261                  self.__monitor.push_level(policy) 
262                  try: 
263                      results = yield generator 
264                  finally: 
265                      self.__monitor.pop_level() 
266                  yield results 
267          finally: 
268              self.__streams.remove(key) 
269