Source Code for Module lepl.stream.core

  1   
  2  # The contents of this file are subject to the Mozilla Public License 
  3  # (MPL) Version 1.1 (the "License"); you may not use this file except 
  4  # in compliance with the License. You may obtain a copy of the License 
  5  # at http://www.mozilla.org/MPL/ 
  6  # 
  7  # Software distributed under the License is distributed on an "AS IS" 
  8  # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 
  9  # the License for the specific language governing rights and 
 10  # limitations under the License. 
 11  # 
 12  # The Original Code is LEPL (http://www.acooke.org/lepl) 
 13  # The Initial Developer of the Original Code is Andrew Cooke. 
 14  # Portions created by the Initial Developer are Copyright (C) 2009-2010 
 15  # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 
 16  # 
 17  # Alternatively, the contents of this file may be used under the terms 
 18  # of the LGPL license (the GNU Lesser General Public License, 
 19  # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 
 20  # of the LGPL License are applicable instead of those above. 
 21  # 
 22  # If you wish to allow use of your version of this file only under the 
 23  # terms of the LGPL License and not to allow others to use your version 
 24  # of this file under the MPL, indicate your decision by deleting the 
 25  # provisions above and replace them with the notice and other provisions 
 26  # required by the LGPL License.  If you do not delete the provisions 
 27  # above, a recipient may use your version of this file under either the 
 28  # MPL or the LGPL License. 
 29   
 30  ''' 
 31  Default implementations of the stream classes.  
 32   
 33  A stream is a tuple (state, helper), where `state` will vary from location to  
 34  location, while `helper` is an "unchanging" instance of `StreamHelper`,  
 35  defined below. 
 36   
 37  For simple streams state can be a simple integer and this approach avoids the 
 38  repeated creation of objects.  More complex streams may choose to not use 
 39  the state at all, simply creating a new helper at each point. 
 40  ''' 
 41   
 42  from abc import ABCMeta 
 43   
 44  from lepl.support.lib import fmt 
 45   
 46   
 47  #class _SimpleStream(metaclass=ABCMeta): 
 48  # Python 2.6 
 49  # pylint: disable-msg=W0105, C0103 
 50  _StreamHelper = ABCMeta('_StreamHelper', (object, ), {}) 
 51  '''ABC used to identify streams.''' 
 52   
 53  DUMMY_HELPER = object() 
 54  '''Allows tests to specify an arbitrary helper in results.''' 
 55       
 56  OFFSET, LINENO, CHAR = range(3) 
 57  '''Indices into delta.''' 
 58   
 59   
 60 -class StreamHelper(_StreamHelper): 
 61      ''' 
 62      The interface that all helpers should implement. 
 63      ''' 
 64       
 65 -    def __init__(self, id=None, factory=None, max=None, global_kargs=None, 
 66                   cache_level=None): 
 67          from lepl.stream.factory import DEFAULT_STREAM_FACTORY 
 68          self.id = id if id is not None else hash(self) 
 69          self.factory = factory if factory else DEFAULT_STREAM_FACTORY 
 70          self.max = max if max else MutableMaxDepth() 
 71          self.global_kargs = global_kargs if global_kargs else {} 
 72          self.cache_level = 1 if cache_level is None else cache_level 
 73       
 74 -    def __repr__(self): 
 75          '''Simplify for comparison in tests''' 
 76          return '<helper>' 
 77       
 78 -    def __eq__(self, other): 
 79          return other is DUMMY_HELPER or super(StreamHelper, self).__eq__(other) 
 80       
 81 -    def __hash__(self): 
 82          return super(StreamHelper, self).__hash__() 
 83   
 84 -    def key(self, state, other): 
 85          ''' 
 86          Generate an object that can be hashed (implements __hash__ and __eq__). 
 87          See `HashKey`. 
 88          ''' 
 89          raise NotImplementedError 
 90       
 91 -    def kargs(self, state, prefix='', kargs=None): 
 92          ''' 
 93          Generate a dictionary of values that describe the stream.  These 
 94          may be extended by subclasses.  They are provided to  
 95          `syntax_error_kargs`, for example. 
 96           
 97          `prefix` modifies the property names 
 98           
 99          `kargs` allows values to be provided.  These are *not* overwritten, 
100          so if there is a name clash the provided value remains. 
101           
102          Note: Calculating this can be expensive; use only for error messages, 
103          not debug messages (that may be discarded). 
104           
105          The following names will be defined (at a minimum). 
106           
107          For these value the "global" prefix indicates the underlying stream  
108          when, for example, tokens are used (other values will be relative to  
109          the token).  If tokens etc are not in use then global and non-global  
110          values will agree. 
111          - data: a line representing the data, highlighting the current offset 
112          - global_data: as data, but for the entire sequence 
113          - text: as data, but without a "[...]" at the end 
114          - global_text: as text, but for the entire sequence 
115          - type: the type of the sequence 
116          - global_type: the type of the entire sequence 
117          - global_offset: a 0-based index into the underlying sequence 
118   
119          These values are always local: 
120          - offset: a 0-based index into the sequence 
121          - rest: the data following the current point 
122          - repr: the current value, or <EOS> 
123          - str: the current value, or an empty string 
124           
125          These values are always global: 
126          - filename: a filename, if available, or the type 
127          - lineno: a 1-based line number for the current offset 
128          - char: a 1-based character count within the line for the current offset 
129          - location: a summary of the current location 
130          ''' 
131          raise NotImplementedError 
132   
133 -    def fmt(self, state, template, prefix='', kargs=None): 
134          '''fmt a message using the expensive kargs function.''' 
135          return fmt(template, **self.kargs(state, prefix=prefix, kargs=kargs)) 
136       
137 -    def debug(self, state): 
138          '''Generate an inexpensive debug message.''' 
139          raise NotImplementedError 
140       
141 -    def next(self, state, count=1): 
142          ''' 
143          Return (value, stream) where `value` is the next value (or  
144          values if count > 1) from the stream and `stream` is advanced to the 
145          next character.  Note that `value` is always a sequence (so if the  
146          stream is a list of integers, and `count`=1, then it will be a  
147          unitary list, for example). 
148           
149          Should raise StopIteration when no more data are available. 
150          ''' 
151          raise StopIteration 
152       
153 -    def join(self, state, *values): 
154          ''' 
155          Join sequences of values into a single sequence. 
156          ''' 
157          raise NotImplementedError 
158       
159 -    def empty(self, state): 
160          ''' 
161          Return true if no more data available. 
162          ''' 
163          raise NotImplementedError 
164       
165 -    def line(self, state, empty_ok): 
166          ''' 
167          Return (values, stream) where `values` correspond to something 
168          like "the rest of the line" from the current point and `stream` 
169          is advanced to the point after the line ends. 
170           
171          If `empty_ok` is true and we are at the end of a line, return an 
172          empty line, otherwise advance (and maybe raise a StopIteration). 
173          ''' 
174          raise NotImplementedError 
175       
176 -    def len(self, state): 
177          ''' 
178          Return the remaining length of the stream.  Streams of unknown 
179          length (iterables) should raise a TypeError. 
180          ''' 
181          raise NotImplementedError 
182       
183 -    def stream(self, state, value, id_=None, max=None): 
184          ''' 
185          Return a new stream that encapsulates the value given, starting at 
186          `state`.  IMPORTANT: the stream used is the one that corresponds to 
187          the start of the value. 
188             
189          For example: 
190              (line, next_stream) = s_line(stream, False) 
191              token_stream = s_stream(stream, line) # uses stream, not next_stream 
192            
193          This is used when processing Tokens, for example, or columns (where 
194          fragments in the correct column area are parsed separately). 
195          ''' 
196          raise NotImplementedError 
197       
198 -    def deepest(self): 
199          ''' 
200          Return a stream that represents the deepest match.  The stream may be 
201          incomplete in some sense (it may not be possible to use it for 
202          parsing more data), but it will have usable fmt and kargs methods. 
203          ''' 
204          raise NotImplementedError 
205       
206 -    def delta(self, state): 
207          ''' 
208          Return the offset, lineno and char of the current point, relative to  
209          the entire stream, as a tuple.  
210          ''' 
211          raise NotImplementedError 
212       
213 -    def eq(self, state1, state2): 
214          ''' 
215          Are the two states equal? 
216          ''' 
217          return state1 == state2 
218       
219 -    def new_max(self, state): 
220          ''' 
221          Return (old max, new stream), where new stream uses a new max. 
222          This is used when we want to read from the stream without  
223          affecting the max (eg when looking ahead to generate tokens). 
224          ''' 
225          raise NotImplementedError 
226       
227 -    def cacheable(self): 
228          ''' 
229          Is this stream cacheable? 
230          ''' 
231          return self.cache_level > 0 
232       
233   
234  # The following are helper functions that allow the methods above to be 
235  # called on (state, helper) tuples 
236   
237  s_key = lambda stream, other=None: stream[1].key(stream[0], other) 
238  '''Invoke helper.key(state, other)''' 
239   
240  s_kargs = lambda stream, prefix='', kargs=None: stream[1].kargs(stream[0], prefix=prefix, kargs=kargs) 
241  '''Invoke helper.kargs(state, prefix, kargs)''' 
242   
243  s_fmt = lambda stream, template, prefix='', kargs=None: stream[1].fmt(stream[0], template, prefix=prefix, kargs=kargs) 
244  '''Invoke helper.fmt(state, template, prefix, kargs)''' 
245   
246  s_debug = lambda stream: stream[1].debug(stream[0]) 
247  '''Invoke helper.debug()''' 
248   
249  s_next = lambda stream, count=1: stream[1].next(stream[0], count=count) 
250  '''Invoke helper.next(state, count)''' 
251   
252  s_join = lambda stream, *values: stream[1].join(stream[0], *values) 
253  '''Invoke helper.join(*values)''' 
254   
255  s_empty = lambda stream: stream[1].empty(stream[0]) 
256  '''Invoke helper.empty(state)''' 
257   
258  s_line = lambda stream, empty_ok: stream[1].line(stream[0], empty_ok) 
259  '''Invoke helper.line(state, empty_ok)''' 
260   
261  s_len = lambda stream: stream[1].len(stream[0]) 
262  '''Invoke helper.len(state)''' 
263   
264  s_stream = lambda stream, value, id_=None, max=None: stream[1].stream(stream[0], value, id_=id_, max=max) 
265  '''Invoke helper.stream(state, value)''' 
266   
267  s_deepest = lambda stream: stream[1].deepest() 
268  '''Invoke helper.deepest()''' 
269   
270  s_delta = lambda stream: stream[1].delta(stream[0]) 
271  '''Invoke helper.delta(state)''' 
272   
273  s_eq = lambda stream1, stream2: stream1[1].eq(stream1[0], stream2[0]) 
274  '''Compare two streams (which should have identical helpers)''' 
275   
276  s_id = lambda stream: stream[1].id 
277  '''Access the ID attribute.''' 
278   
279  s_factory = lambda stream: stream[1].factory 
280  '''Access the factory attribute.''' 
281   
282  s_max = lambda stream: stream[1].max 
283  '''Access the max attribute.''' 
284   
285  s_new_max = lambda stream: stream[1].new_max(stream[0]) 
286  '''Invoke helper.new_max(state).''' 
287   
288  s_global_kargs = lambda stream: stream[1].global_kargs 
289  '''Access the global_kargs attribute.''' 
290   
291  s_cache_level = lambda stream: stream[1].cache_level 
292  '''Access the cache_level attribute.''' 
293   
294  s_cacheable = lambda stream: stream[1].cacheable() 
295  '''Is the stream cacheable?''' 
296   
297   
298 -class MutableMaxDepth(object): 
299      ''' 
300      Track maximum depth (offset) reached and the associated stream.  Used to 
301      generate error message for incomplete matches. 
302      ''' 
303       
304 -    def __init__(self): 
305          self.depth = 0 
306          self.stream = None 
307           
308 -    def update(self, depth, stream): 
309          # the '=' here allows a token to nudge on to the next stream without 
310          # changing the offset (when count=0 in s_next) 
311          if depth >= self.depth or not self.stream: 
312              self.depth = depth 
313              self.stream = stream 
314           
315 -    def get(self): 
316          return self.stream 
317       
318   
319 -class HashKey(object): 
320      ''' 
321      Used to store a value with a given hash. 
322      ''' 
323       
324      __slots__ = ['hash', 'eq'] 
325       
326 -    def __init__(self, hash, eq=None): 
327          self.hash = hash 
328          self.eq = eq 
329           
330 -    def __hash__(self): 
331          return self.hash 
332       
333 -    def __eq__(self, other): 
334          try: 
335              return other.hash == self.hash and other.eq == self.eq 
336          except AttributeError: 
337              return False 
338