Package lepl :: Package stream :: Module iter
[hide private]
[frames] | no frames]

Source Code for Module lepl.stream.iter

  1   
  2  # The contents of this file are subject to the Mozilla Public License 
  3  # (MPL) Version 1.1 (the "License"); you may not use this file except 
  4  # in compliance with the License. You may obtain a copy of the License 
  5  # at http://www.mozilla.org/MPL/ 
  6  # 
  7  # Software distributed under the License is distributed on an "AS IS" 
  8  # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 
  9  # the License for the specific language governing rights and 
 10  # limitations under the License. 
 11  # 
 12  # The Original Code is LEPL (http://www.acooke.org/lepl) 
 13  # The Initial Developer of the Original Code is Andrew Cooke. 
 14  # Portions created by the Initial Developer are Copyright (C) 2009-2010 
 15  # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 
 16  # 
 17  # Alternatively, the contents of this file may be used under the terms 
 18  # of the LGPL license (the GNU Lesser General Public License, 
 19  # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 
 20  # of the LGPL License are applicable instead of those above. 
 21  # 
 22  # If you wish to allow use of your version of this file only under the 
 23  # terms of the LGPL License and not to allow others to use your version 
 24  # of this file under the MPL, indicate your decision by deleting the 
 25  # provisions above and replace them with the notice and other provisions 
 26  # required by the LGPL License.  If you do not delete the provisions 
 27  # above, a recipient may use your version of this file under either the 
 28  # MPL or the LGPL License. 
 29   
 30  ''' 
 31  A stream for iterable sources.  Each value in the iteration is considered as  
 32  a line (which makes sense for files, for example, which iterate over lines). 
 33   
 34  The source is wrapped in a `Cons` object.  This has an attribute `head` 
 35  which contains the current line and a method `tail()` which returns another 
 36  `Cons` instance, or raise a `StopIteration`. 
 37   
 38  The stream has the form `(state, helper)`, where `helper` is an  
 39  `IterableHelper` instance, as described below. 
 40   
 41  The `state` value in the stream described above has the form 
 42  `(cons, line_stream)` where `cons` is a `Cons` instance and line_stream 
 43  is a stream generated from `cons.head` (so has the structure (state', helper') 
 44  where state' and helper' depend on the type of the line and the stream factory 
 45  used). 
 46   
 47  Evaluation of stream methods then typically has the form: 
 48  - call to IterableHelper 
 49  - unpacking of state 
 50  - delegation to line_stream 
 51  - possible exception handling  
 52   
 53  This has the  advantages of being generic in the type returned by the 
 54  iterator, of being customizable (by specifying a new factory), and re-using 
 55  existing code where possible (in the use of the sub-helper).  It should even 
 56  be possible to have iterables of iterables... 
 57  ''' 
 58   
 59  from lepl.support.lib import add_defaults, fmt 
 60  from lepl.stream.simple import OFFSET, LINENO, BaseHelper 
 61  from lepl.stream.core import s_delta, s_kargs, s_fmt, s_debug, s_next, \ 
 62      s_line, s_join, s_empty, s_eq, HashKey 
63 64 65 -class Cons(object):
66 ''' 67 A linked list cell that is a lazy wrapper around an iterable. So "tail" 68 returns the next iterable on demand. 69 ''' 70 71 __slots__ = ['_iterable', '_head', '_tail', '_expanded'] 72
73 - def __init__(self, iterable):
74 self._iterable = iterable 75 self._head = None 76 self._tail = None 77 self._expanded = False
78
79 - def _expand(self):
80 if not self._expanded: 81 self._head = next(self._iterable) 82 self._tail = Cons(self._iterable) 83 self._expanded = True
84 85 @property
86 - def head(self):
87 self._expand() 88 return self._head
89 90 @property
91 - def tail(self):
92 self._expand() 93 return self._tail
94
95 96 -def base_iterable_factory(state_to_line_stream, type_):
97 ''' 98 `IterableHelper` and the token helper differ mainly in how they map from 99 `state` to `line_stream`. 100 ''' 101 102 class BaseIterableHelper(BaseHelper): 103 104 def __init__(self, id=None, factory=None, max=None, global_kargs=None, 105 cache_level=None, delta=None): 106 super(BaseIterableHelper, self).__init__(id=id, factory=factory, 107 max=max, global_kargs=global_kargs, 108 cache_level=cache_level, delta=delta) 109 add_defaults(self.global_kargs, { 110 'global_type': type_, 111 'filename': type_}) 112 self._kargs = dict(self.global_kargs) 113 add_defaults(self._kargs, {'type': type_})
114 115 def key(self, state, other): 116 try: 117 line_stream = state_to_line_stream(state) 118 offset = s_delta(line_stream)[OFFSET] 119 except StopIteration: 120 self._warn('Default hash') 121 offset = -1 122 key = HashKey(self.id ^ offset ^ hash(other), (self.id, other)) 123 #self._debug(fmt('Hash at {0!r} ({1}): {2}', state, offset, hash(key))) 124 return key 125 126 def kargs(self, state, prefix='', kargs=None): 127 line_stream = state_to_line_stream(state) 128 return s_kargs(line_stream, prefix=prefix, kargs=kargs) 129 130 def fmt(self, state, template, prefix='', kargs=None): 131 line_stream = state_to_line_stream(state) 132 return s_fmt(line_stream, template, prefix=prefix, kargs=kargs) 133 134 def debug(self, state): 135 try: 136 line_stream = state_to_line_stream(state) 137 return s_debug(line_stream) 138 except StopIteration: 139 return '<EOS>' 140 141 def join(self, state, *values): 142 line_stream = state_to_line_stream(state) 143 return s_join(line_stream, *values) 144 145 def empty(self, state): 146 try: 147 self.next(state) 148 return False 149 except StopIteration: 150 return True 151 152 def delta(self, state): 153 line_stream = state_to_line_stream(state) 154 return s_delta(line_stream) 155 156 def eq(self, state1, state2): 157 line_stream1 = state_to_line_stream(state1) 158 line_stream2 = state_to_line_stream(state2) 159 return s_eq(line_stream1, line_stream2) 160 161 def deepest(self): 162 return self.max.get() 163 164 def new_max(self, state): 165 return (self.max, 166 (state, type(self)(id=self.id, factory=self.factory, 167 max=None, delta=self.delta, 168 global_kargs=self.global_kargs, 169 cache_level=self.cache_level))) 170 171 return BaseIterableHelper 172
173 174 -class IterableHelper( 175 base_iterable_factory(lambda state: state[1], '<iterable>')):
176 ''' 177 Implement a stream over iterable values. 178 ''' 179
180 - def _next_line(self, cons, empty_line_stream):
181 delta = s_delta(empty_line_stream) 182 delta = (delta[OFFSET], delta[LINENO]+1, 1) 183 return self.factory(cons.head, id=self.id, factory=self.factory, 184 max=self.max, global_kargs=self.global_kargs, 185 delta=delta)
186
187 - def next(self, state, count=1):
188 (cons, line_stream) = state 189 try: 190 (value, next_line_stream) = s_next(line_stream, count=count) 191 return (value, ((cons, next_line_stream), self)) 192 except StopIteration: 193 # the general approach here is to take what we can from the 194 # current line, create the next, and take the rest from that. 195 # of course, that may also not have enough, in which case it 196 # will recurse. 197 cons = cons.tail 198 if s_empty(line_stream): 199 next_line_stream = self._next_line(cons, line_stream) 200 next_stream = ((cons, next_line_stream), self) 201 return s_next(next_stream, count=count) 202 else: 203 (line, end_line_stream) = s_line(line_stream, False) 204 next_line_stream = self._next_line(cons, end_line_stream) 205 next_stream = ((cons, next_line_stream), self) 206 (extra, final_stream) = s_next(next_stream, count=count-len(line)) 207 value = s_join(line_stream, line, extra) 208 return (value, final_stream)
209
210 - def line(self, state, empty_ok):
211 try: 212 (cons, line_stream) = state 213 if s_empty(line_stream): 214 cons = cons.tail 215 line_stream = self._next_line(cons, line_stream) 216 (value, empty_line_stream) = s_line(line_stream, empty_ok) 217 return (value, ((cons, empty_line_stream), self)) 218 except StopIteration: 219 if empty_ok: 220 raise TypeError('Iterable stream cannot return an empty line') 221 else: 222 raise
223
224 - def len(self, state):
225 self._error('len(iter)') 226 raise TypeError
227
228 - def stream(self, state, value, id_=None, max=None):
229 (cons, line_stream) = state 230 id_ = self.id if id_ is None else id_ 231 max = max if max else self.max 232 next_line_stream = \ 233 self.factory(value, id=id_, factory=self.factory, max=max, 234 global_kargs=self.global_kargs, 235 cache_level=self.cache_level+1, 236 delta=s_delta(line_stream)) 237 return ((cons, next_line_stream), self)
238