Package lepl :: Package stream :: Module core
[hide private]
[frames] | no frames]

Source Code for Module lepl.stream.core

  1   
  2  # The contents of this file are subject to the Mozilla Public License 
  3  # (MPL) Version 1.1 (the "License"); you may not use this file except 
  4  # in compliance with the License. You may obtain a copy of the License 
  5  # at http://www.mozilla.org/MPL/ 
  6  # 
  7  # Software distributed under the License is distributed on an "AS IS" 
  8  # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 
  9  # the License for the specific language governing rights and 
 10  # limitations under the License. 
 11  # 
 12  # The Original Code is LEPL (http://www.acooke.org/lepl) 
 13  # The Initial Developer of the Original Code is Andrew Cooke. 
 14  # Portions created by the Initial Developer are Copyright (C) 2009-2010 
 15  # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 
 16  # 
 17  # Alternatively, the contents of this file may be used under the terms 
 18  # of the LGPL license (the GNU Lesser General Public License, 
 19  # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 
 20  # of the LGPL License are applicable instead of those above. 
 21  # 
 22  # If you wish to allow use of your version of this file only under the 
 23  # terms of the LGPL License and not to allow others to use your version 
 24  # of this file under the MPL, indicate your decision by deleting the 
 25  # provisions above and replace them with the notice and other provisions 
 26  # required by the LGPL License.  If you do not delete the provisions 
 27  # above, a recipient may use your version of this file under either the 
 28  # MPL or the LGPL License. 
 29   
 30  ''' 
 31  Default implementations of the stream classes.  
 32   
 33  A stream is a tuple (state, helper), where `state` will vary from location to  
 34  location, while `helper` is an "unchanging" instance of `StreamHelper`,  
 35  defined below. 
 36   
 37  For simple streams state can be a simple integer and this approach avoids the 
 38  repeated creation of objects.  More complex streams may choose to not use 
 39  the state at all, simply creating a new helper at each point. 
 40  ''' 
 41   
 42  from abc import ABCMeta 
 43   
 44  from lepl.support.lib import fmt 
 45   
 46   
 47  #class _SimpleStream(metaclass=ABCMeta): 
 48  # Python 2.6 
 49  # pylint: disable-msg=W0105, C0103 
 50  _StreamHelper = ABCMeta('_StreamHelper', (object, ), {}) 
 51  '''ABC used to identify streams.''' 
 52   
 53  DUMMY_HELPER = object() 
 54  '''Allows tests to specify an arbitrary helper in results.''' 
 55       
 56  OFFSET, LINENO, CHAR = range(3) 
 57  '''Indices into delta.''' 
 58   
 59   
60 -class StreamHelper(_StreamHelper):
61 ''' 62 The interface that all helpers should implement. 63 ''' 64
65 - def __init__(self, id=None, factory=None, max=None, global_kargs=None, 66 cache_level=None):
67 from lepl.stream.factory import DEFAULT_STREAM_FACTORY 68 self.id = id if id is not None else hash(self) 69 self.factory = factory if factory else DEFAULT_STREAM_FACTORY 70 self.max = max if max else MutableMaxDepth() 71 self.global_kargs = global_kargs if global_kargs else {} 72 self.cache_level = 1 if cache_level is None else cache_level
73
74 - def __repr__(self):
75 '''Simplify for comparison in tests''' 76 return '<helper>'
77
78 - def __eq__(self, other):
79 return other is DUMMY_HELPER or super(StreamHelper, self).__eq__(other)
80
81 - def __hash__(self):
82 return super(StreamHelper, self).__hash__()
83
84 - def key(self, state, other):
85 ''' 86 Generate an object that can be hashed (implements __hash__ and __eq__). 87 See `HashKey`. 88 ''' 89 raise NotImplementedError
90
91 - def kargs(self, state, prefix='', kargs=None):
92 ''' 93 Generate a dictionary of values that describe the stream. These 94 may be extended by subclasses. They are provided to 95 `syntax_error_kargs`, for example. 96 97 `prefix` modifies the property names 98 99 `kargs` allows values to be provided. These are *not* overwritten, 100 so if there is a name clash the provided value remains. 101 102 Note: Calculating this can be expensive; use only for error messages, 103 not debug messages (that may be discarded). 104 105 The following names will be defined (at a minimum). 106 107 For these value the "global" prefix indicates the underlying stream 108 when, for example, tokens are used (other values will be relative to 109 the token). If tokens etc are not in use then global and non-global 110 values will agree. 111 - data: a line representing the data, highlighting the current offset 112 - global_data: as data, but for the entire sequence 113 - text: as data, but without a "[...]" at the end 114 - global_text: as text, but for the entire sequence 115 - type: the type of the sequence 116 - global_type: the type of the entire sequence 117 - global_offset: a 0-based index into the underlying sequence 118 119 These values are always local: 120 - offset: a 0-based index into the sequence 121 - rest: the data following the current point 122 - repr: the current value, or <EOS> 123 - str: the current value, or an empty string 124 125 These values are always global: 126 - filename: a filename, if available, or the type 127 - lineno: a 1-based line number for the current offset 128 - char: a 1-based character count within the line for the current offset 129 - location: a summary of the current location 130 ''' 131 raise NotImplementedError
132
133 - def fmt(self, state, template, prefix='', kargs=None):
134 '''fmt a message using the expensive kargs function.''' 135 return fmt(template, **self.kargs(state, prefix=prefix, kargs=kargs))
136
137 - def debug(self, state):
138 '''Generate an inexpensive debug message.''' 139 raise NotImplementedError
140
141 - def next(self, state, count=1):
142 ''' 143 Return (value, stream) where `value` is the next value (or 144 values if count > 1) from the stream and `stream` is advanced to the 145 next character. Note that `value` is always a sequence (so if the 146 stream is a list of integers, and `count`=1, then it will be a 147 unitary list, for example). 148 149 Should raise StopIteration when no more data are available. 150 ''' 151 raise StopIteration
152
153 - def join(self, state, *values):
154 ''' 155 Join sequences of values into a single sequence. 156 ''' 157 raise NotImplementedError
158
159 - def empty(self, state):
160 ''' 161 Return true if no more data available. 162 ''' 163 raise NotImplementedError
164
165 - def line(self, state, empty_ok):
166 ''' 167 Return (values, stream) where `values` correspond to something 168 like "the rest of the line" from the current point and `stream` 169 is advanced to the point after the line ends. 170 171 If `empty_ok` is true and we are at the end of a line, return an 172 empty line, otherwise advance (and maybe raise a StopIteration). 173 ''' 174 raise NotImplementedError
175
176 - def len(self, state):
177 ''' 178 Return the remaining length of the stream. Streams of unknown 179 length (iterables) should raise a TypeError. 180 ''' 181 raise NotImplementedError
182
183 - def stream(self, state, value, id_=None, max=None):
184 ''' 185 Return a new stream that encapsulates the value given, starting at 186 `state`. IMPORTANT: the stream used is the one that corresponds to 187 the start of the value. 188 189 For example: 190 (line, next_stream) = s_line(stream, False) 191 token_stream = s_stream(stream, line) # uses stream, not next_stream 192 193 This is used when processing Tokens, for example, or columns (where 194 fragments in the correct column area are parsed separately). 195 ''' 196 raise NotImplementedError
197
198 - def deepest(self):
199 ''' 200 Return a stream that represents the deepest match. The stream may be 201 incomplete in some sense (it may not be possible to use it for 202 parsing more data), but it will have usable fmt and kargs methods. 203 ''' 204 raise NotImplementedError
205
206 - def delta(self, state):
207 ''' 208 Return the offset, lineno and char of the current point, relative to 209 the entire stream, as a tuple. 210 ''' 211 raise NotImplementedError
212
213 - def eq(self, state1, state2):
214 ''' 215 Are the two states equal? 216 ''' 217 return state1 == state2
218
219 - def new_max(self, state):
220 ''' 221 Return (old max, new stream), where new stream uses a new max. 222 This is used when we want to read from the stream without 223 affecting the max (eg when looking ahead to generate tokens). 224 ''' 225 raise NotImplementedError
226
227 - def cacheable(self):
228 ''' 229 Is this stream cacheable? 230 ''' 231 return self.cache_level > 0
232 233 234 # The following are helper functions that allow the methods above to be 235 # called on (state, helper) tuples 236 237 s_key = lambda stream, other=None: stream[1].key(stream[0], other) 238 '''Invoke helper.key(state, other)''' 239 240 s_kargs = lambda stream, prefix='', kargs=None: stream[1].kargs(stream[0], prefix=prefix, kargs=kargs) 241 '''Invoke helper.kargs(state, prefix, kargs)''' 242 243 s_fmt = lambda stream, template, prefix='', kargs=None: stream[1].fmt(stream[0], template, prefix=prefix, kargs=kargs) 244 '''Invoke helper.fmt(state, template, prefix, kargs)''' 245 246 s_debug = lambda stream: stream[1].debug(stream[0]) 247 '''Invoke helper.debug()''' 248 249 s_next = lambda stream, count=1: stream[1].next(stream[0], count=count) 250 '''Invoke helper.next(state, count)''' 251 252 s_join = lambda stream, *values: stream[1].join(stream[0], *values) 253 '''Invoke helper.join(*values)''' 254 255 s_empty = lambda stream: stream[1].empty(stream[0]) 256 '''Invoke helper.empty(state)''' 257 258 s_line = lambda stream, empty_ok: stream[1].line(stream[0], empty_ok) 259 '''Invoke helper.line(state, empty_ok)''' 260 261 s_len = lambda stream: stream[1].len(stream[0]) 262 '''Invoke helper.len(state)''' 263 264 s_stream = lambda stream, value, id_=None, max=None: stream[1].stream(stream[0], value, id_=id_, max=max) 265 '''Invoke helper.stream(state, value)''' 266 267 s_deepest = lambda stream: stream[1].deepest() 268 '''Invoke helper.deepest()''' 269 270 s_delta = lambda stream: stream[1].delta(stream[0]) 271 '''Invoke helper.delta(state)''' 272 273 s_eq = lambda stream1, stream2: stream1[1].eq(stream1[0], stream2[0]) 274 '''Compare two streams (which should have identical helpers)''' 275 276 s_id = lambda stream: stream[1].id 277 '''Access the ID attribute.''' 278 279 s_factory = lambda stream: stream[1].factory 280 '''Access the factory attribute.''' 281 282 s_max = lambda stream: stream[1].max 283 '''Access the max attribute.''' 284 285 s_new_max = lambda stream: stream[1].new_max(stream[0]) 286 '''Invoke helper.new_max(state).''' 287 288 s_global_kargs = lambda stream: stream[1].global_kargs 289 '''Access the global_kargs attribute.''' 290 291 s_cache_level = lambda stream: stream[1].cache_level 292 '''Access the cache_level attribute.''' 293 294 s_cacheable = lambda stream: stream[1].cacheable() 295 '''Is the stream cacheable?''' 296 297
298 -class MutableMaxDepth(object):
299 ''' 300 Track maximum depth (offset) reached and the associated stream. Used to 301 generate error message for incomplete matches. 302 ''' 303
304 - def __init__(self):
305 self.depth = 0 306 self.stream = None
307
308 - def update(self, depth, stream):
309 # the '=' here allows a token to nudge on to the next stream without 310 # changing the offset (when count=0 in s_next) 311 if depth >= self.depth or not self.stream: 312 self.depth = depth 313 self.stream = stream
314
315 - def get(self):
316 return self.stream
317 318
319 -class HashKey(object):
320 ''' 321 Used to store a value with a given hash. 322 ''' 323 324 __slots__ = ['hash', 'eq'] 325
326 - def __init__(self, hash, eq=None):
327 self.hash = hash 328 self.eq = eq
329
330 - def __hash__(self):
331 return self.hash
332
333 - def __eq__(self, other):
334 try: 335 return other.hash == self.hash and other.eq == self.eq 336 except AttributeError: 337 return False
338