1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 '''
31 A stream for iterable sources. Each value in the iteration is considered as
32 a line (which makes sense for files, for example, which iterate over lines).
33
34 The source is wrapped in a `Cons` object. This has an attribute `head`
35 which contains the current line and a method `tail()` which returns another
36 `Cons` instance, or raise a `StopIteration`.
37
38 The stream has the form `(state, helper)`, where `helper` is an
39 `IterableHelper` instance, as described below.
40
41 The `state` value in the stream described above has the form
42 `(cons, line_stream)` where `cons` is a `Cons` instance and line_stream
43 is a stream generated from `cons.head` (so has the structure (state', helper')
44 where state' and helper' depend on the type of the line and the stream factory
45 used).
46
47 Evaluation of stream methods then typically has the form:
48 - call to IterableHelper
49 - unpacking of state
50 - delegation to line_stream
51 - possible exception handling
52
53 This has the advantages of being generic in the type returned by the
54 iterator, of being customizable (by specifying a new factory), and re-using
55 existing code where possible (in the use of the sub-helper). It should even
56 be possible to have iterables of iterables...
57 '''
58
59 from lepl.support.lib import add_defaults, fmt
60 from lepl.stream.simple import OFFSET, LINENO, BaseHelper
61 from lepl.stream.core import s_delta, s_kargs, s_fmt, s_debug, s_next, \
62 s_line, s_join, s_empty, s_eq, HashKey
63
64
65 -class Cons(object):
66 '''
67 A linked list cell that is a lazy wrapper around an iterable. So "tail"
68 returns the next iterable on demand.
69 '''
70
71 __slots__ = ['_iterable', '_head', '_tail', '_expanded']
72
74 self._iterable = iterable
75 self._head = None
76 self._tail = None
77 self._expanded = False
78
80 if not self._expanded:
81 self._head = next(self._iterable)
82 self._tail = Cons(self._iterable)
83 self._expanded = True
84
85 @property
87 self._expand()
88 return self._head
89
90 @property
92 self._expand()
93 return self._tail
94
97 '''
98 `IterableHelper` and the token helper differ mainly in how they map from
99 `state` to `line_stream`.
100 '''
101
102 class BaseIterableHelper(BaseHelper):
103
104 def __init__(self, id=None, factory=None, max=None, global_kargs=None,
105 cache_level=None, delta=None):
106 super(BaseIterableHelper, self).__init__(id=id, factory=factory,
107 max=max, global_kargs=global_kargs,
108 cache_level=cache_level, delta=delta)
109 add_defaults(self.global_kargs, {
110 'global_type': type_,
111 'filename': type_})
112 self._kargs = dict(self.global_kargs)
113 add_defaults(self._kargs, {'type': type_})
114
115 def key(self, state, other):
116 try:
117 line_stream = state_to_line_stream(state)
118 offset = s_delta(line_stream)[OFFSET]
119 except StopIteration:
120 self._warn('Default hash')
121 offset = -1
122 key = HashKey(self.id ^ offset ^ hash(other), (self.id, other))
123
124 return key
125
126 def kargs(self, state, prefix='', kargs=None):
127 line_stream = state_to_line_stream(state)
128 return s_kargs(line_stream, prefix=prefix, kargs=kargs)
129
130 def fmt(self, state, template, prefix='', kargs=None):
131 line_stream = state_to_line_stream(state)
132 return s_fmt(line_stream, template, prefix=prefix, kargs=kargs)
133
134 def debug(self, state):
135 try:
136 line_stream = state_to_line_stream(state)
137 return s_debug(line_stream)
138 except StopIteration:
139 return '<EOS>'
140
141 def join(self, state, *values):
142 line_stream = state_to_line_stream(state)
143 return s_join(line_stream, *values)
144
145 def empty(self, state):
146 try:
147 self.next(state)
148 return False
149 except StopIteration:
150 return True
151
152 def delta(self, state):
153 line_stream = state_to_line_stream(state)
154 return s_delta(line_stream)
155
156 def eq(self, state1, state2):
157 line_stream1 = state_to_line_stream(state1)
158 line_stream2 = state_to_line_stream(state2)
159 return s_eq(line_stream1, line_stream2)
160
161 def deepest(self):
162 return self.max.get()
163
164 def new_max(self, state):
165 return (self.max,
166 (state, type(self)(id=self.id, factory=self.factory,
167 max=None, delta=self.delta,
168 global_kargs=self.global_kargs,
169 cache_level=self.cache_level)))
170
171 return BaseIterableHelper
172
176 '''
177 Implement a stream over iterable values.
178 '''
179
186
187 - def next(self, state, count=1):
188 (cons, line_stream) = state
189 try:
190 (value, next_line_stream) = s_next(line_stream, count=count)
191 return (value, ((cons, next_line_stream), self))
192 except StopIteration:
193
194
195
196
197 cons = cons.tail
198 if s_empty(line_stream):
199 next_line_stream = self._next_line(cons, line_stream)
200 next_stream = ((cons, next_line_stream), self)
201 return s_next(next_stream, count=count)
202 else:
203 (line, end_line_stream) = s_line(line_stream, False)
204 next_line_stream = self._next_line(cons, end_line_stream)
205 next_stream = ((cons, next_line_stream), self)
206 (extra, final_stream) = s_next(next_stream, count=count-len(line))
207 value = s_join(line_stream, line, extra)
208 return (value, final_stream)
209
210 - def line(self, state, empty_ok):
211 try:
212 (cons, line_stream) = state
213 if s_empty(line_stream):
214 cons = cons.tail
215 line_stream = self._next_line(cons, line_stream)
216 (value, empty_line_stream) = s_line(line_stream, empty_ok)
217 return (value, ((cons, empty_line_stream), self))
218 except StopIteration:
219 if empty_ok:
220 raise TypeError('Iterable stream cannot return an empty line')
221 else:
222 raise
223
224 - def len(self, state):
225 self._error('len(iter)')
226 raise TypeError
227
228 - def stream(self, state, value, id_=None, max=None):
238