lepl.matchers._test.memo

1 from lepl.matchers.variables import TraceVariables 2 3 # The contents of this file are subject to the Mozilla Public License 4 # (MPL) Version 1.1 (the "License"); you may not use this file except 5 # in compliance with the License. You may obtain a copy of the License 6 # at http://www.mozilla.org/MPL/ 7 # 8 # Software distributed under the License is distributed on an "AS IS" 9 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 10 # the License for the specific language governing rights and 11 # limitations under the License. 12 # 13 # The Original Code is LEPL (http://www.acooke.org/lepl) 14 # The Initial Developer of the Original Code is Andrew Cooke. 15 # Portions created by the Initial Developer are Copyright (C) 2009-2010 16 # Andrew Cooke (andrew@acooke.org). All Rights Reserved. 17 # 18 # Alternatively, the contents of this file may be used under the terms 19 # of the LGPL license (the GNU Lesser General Public License, 20 # http://www.gnu.org/licenses/lgpl.html), in which case the provisions 21 # of the LGPL License are applicable instead of those above. 22 # 23 # If you wish to allow use of your version of this file only under the 24 # terms of the LGPL License and not to allow others to use your version 25 # of this file under the MPL, indicate your decision by deleting the 26 # provisions above and replace them with the notice and other provisions 27 # required by the LGPL License. If you do not delete the provisions 28 # above, a recipient may use your version of this file under either the 29 # MPL or the LGPL License. 30 31 ''' 32 Tests for the lepl.matchers.memo module. 33 ''' 34 35 from logging import basicConfig, DEBUG 36 from time import time 37 from unittest import TestCase 38 39 from lepl import Delayed, Any, Optional, Node, Literals, Eos, Or, Token 40 41 42 # pylint: disable-msg=C0103, C0111, C0301, W0702, C0324, C0102, C0321 43 # (dude this is just a test) 44 45

46 -class MemoTest(TestCase):

47

48 - def test_right(self):

49 50 #basicConfig(level=DEBUG) 51 52 seq = Delayed() 53 letter = Any() 54 seq += letter & Optional(seq) 55 56 #print(seq.tree()) 57 seq.config.clear().right_memoize().trace_stack(True) 58 #seq.config.clear().right_memoize() 59 p = seq.get_match_string() 60 #print(p.matcher.tree()) 61 62 results = list(p('ab')) 63 assert len(results) == 2, len(results) 64 assert results[0][0] == ['a', 'b'], results[0][0] 65 assert results[1][0] == ['a'], results[1][0]

66 67

68 - def test_left1a(self):

69 70 #basicConfig(level=DEBUG) 71 72 seq = Delayed() 73 letter = Any() 74 seq += Optional(seq) & letter 75 76 seq.config.clear().left_memoize().trace_stack(True) 77 p = seq.get_match() 78 #print(p.matcher) 79 results = list(p('ab')) 80 assert len(results) == 2, len(results) 81 assert results[0][0] == ['a', 'b'], results[0][0] 82 assert results[1][0] == ['a'], results[1][0]

83 84

85 - def test_left1b(self):

86 87 #basicConfig(level=DEBUG) 88 89 seq = Delayed() 90 letter = Any() 91 seq += Optional(seq) & letter 92 93 seq.config.clear().left_memoize().trace_stack(True) 94 p = seq.get_match_string() 95 results = list(p('ab')) 96 assert len(results) == 2, len(results) 97 assert results[0][0] == ['a', 'b'], results[0][0] 98 assert results[1][0] == ['a'], results[1][0]

99 100

101 - def test_left2(self):

102 103 #basicConfig(level=DEBUG) 104 105 seq = Delayed() 106 letter = Any() 107 seq += letter | (seq & letter) 108 109 seq.config.clear().left_memoize().trace_stack(True) 110 p = seq.get_match_string() 111 results = list(p('abcdef')) 112 assert len(results) == 6, len(results) 113 assert results[0][0] == ['a'], results[0][0] 114 assert results[1][0] == ['a', 'b'], results[1][0]

115 116

117 - def test_complex(self):

118 119 #basicConfig(level=DEBUG) 120 121 class VerbPhrase(Node): pass 122 class DetPhrase(Node): pass 123 class SimpleTp(Node): pass 124 class TermPhrase(Node): pass 125 class Sentence(Node): pass 126 127 verb = Literals('knows', 'respects', 'loves') > 'verb' 128 join = Literals('and', 'or') > 'join' 129 proper_noun = Literals('helen', 'john', 'pat') > 'proper_noun' 130 determiner = Literals('every', 'some') > 'determiner' 131 noun = Literals('boy', 'girl', 'man', 'woman') > 'noun' 132 133 verbphrase = Delayed() 134 verbphrase += verb | (verbphrase // join // verbphrase) > VerbPhrase 135 det_phrase = determiner // noun > DetPhrase 136 simple_tp = proper_noun | det_phrase > SimpleTp 137 termphrase = Delayed() 138 termphrase += simple_tp | (termphrase // join // termphrase) > TermPhrase 139 sentence = termphrase // verbphrase // termphrase & Eos() > Sentence 140 141 sentence.config.clear().left_memoize() 142 p = sentence.get_match_string() 143 print(p.matcher.tree()) 144 145 text = 'every boy or some girl and helen and john or pat knows ' \ 146 'and respects or loves every boy or some girl and pat or ' \ 147 'john and helen' 148 # text = 'every boy loves helen' 149 count = 0 150 for _meaning in p(text): 151 count += 1 152 if count < 3: 153 #print(_meaning[0][0]) 154 pass 155 #print(count) 156 assert count == 392, count

157 158

159 -class RecursionTest(TestCase):

160

161 - def right(self):

162 matcher = Delayed() 163 matcher += Any() & Optional(matcher) 164 return matcher

165

166 - def right_token(self, contents=False):

167 matcher = Delayed() 168 inner = Token(Any()) 169 if contents: 170 inner = inner(Or('a', 'b')) 171 matcher += inner & Optional(matcher) 172 return matcher

173

174 - def left(self):

175 matcher = Delayed() 176 matcher += Optional(matcher) & Any() 177 return matcher

178

179 - def left_token(self, contents=False):

180 matcher = Delayed() 181 inner = Token(Any()) 182 if contents: 183 inner = inner(Or('a', 'b')) 184 matcher += Optional(matcher) & inner 185 return matcher

186

187 - def do_test(self, parser):

188 result = parser('ab') 189 assert result == ['a', 'b'], result 190 result = parser('aa') 191 assert result == ['a', 'a'], result 192 result = parser('aaa') 193 assert result == ['a', 'a', 'a'], result 194 result = parser('aba') 195 assert result == ['a', 'b', 'a'], result

196

197 - def test_right_string(self):

198 #basicConfig(level=DEBUG) 199 matcher = self.right() 200 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 201 self.do_test(matcher.get_parse_string())

202

203 - def test_right_null(self):

204 matcher = self.right() 205 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 206 self.do_test(matcher.get_parse())

207

208 - def test_right_token_string(self):

209 #basicConfig(level=DEBUG) 210 matcher = self.right_token() 211 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 212 self.do_test(matcher.get_parse_string())

213

214 - def test_right_token_null(self):

215 #basicConfig(level=DEBUG) 216 matcher = self.right_token() 217 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 218 #matcher.config.no_full_first_match().auto_memoize(full=True) 219 self.do_test(matcher.get_parse())

220

221 - def test_right_token_string_content(self):

222 #basicConfig(level=DEBUG) 223 matcher = self.right_token(True) 224 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 225 self.do_test(matcher.get_parse_string())

226

227 - def test_right_token_null_content(self):

228 #basicConfig(level=DEBUG) 229 matcher = self.right_token(True) 230 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 231 self.do_test(matcher.get_parse())

232

233 - def test_right_string_clear(self):

234 matcher = self.right() 235 matcher.config.clear().auto_memoize(full=True).trace_stack(True) 236 self.do_test(matcher.get_parse_string())

237

238 - def test_right_null_clear(self):

239 matcher = self.right() 240 matcher.config.clear().auto_memoize(full=True).trace_stack(True) 241 self.do_test(matcher.get_parse())

242

243 - def test_right_token_string_clear(self):

244 #basicConfig(level=DEBUG) 245 matcher = self.right_token() 246 matcher.config.clear().auto_memoize(full=True).lexer().trace_stack(True) 247 self.do_test(matcher.get_parse_string())

248

249 - def test_right_token_null_clear(self):

250 #basicConfig(level=DEBUG) 251 matcher = self.right_token() 252 matcher.config.clear().auto_memoize(full=True).lexer().trace_stack(True) 253 self.do_test(matcher.get_parse())

254

255 - def test_right_token_string_clear_content(self):

256 #basicConfig(level=DEBUG) 257 matcher = self.right_token(True) 258 matcher.config.clear().auto_memoize(full=True).lexer().trace_stack(True) 259 self.do_test(matcher.get_parse_string())

260

261 - def test_right_token_null_clear_content(self):

262 #basicConfig(level=DEBUG) 263 matcher = self.right_token(True) 264 matcher.config.clear().auto_memoize(full=True).lexer().trace_stack(True) 265 self.do_test(matcher.get_parse())

266

267 - def test_left_string(self):

268 #basicConfig(level=DEBUG) 269 matcher = self.left() 270 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 271 #self.do_test(lambda stream: list(matcher.parse_string_all(stream))) 272 self.do_test(matcher.get_parse())

273

274 - def test_left_null(self):

275 matcher = self.left() 276 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 277 self.do_test(matcher.get_parse())

278

279 - def test_left_token_string(self):

280 matcher = self.left_token() 281 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 282 self.do_test(matcher.get_parse_string())

283

284 - def test_left_token_null(self):

285 matcher = self.left_token() 286 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 287 self.do_test(matcher.get_parse())

288

289 - def test_left_token_string_content(self):

290 matcher = self.left_token(True) 291 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 292 self.do_test(matcher.get_parse_string())

293

294 - def test_left_token_null_content(self):

295 matcher = self.left_token(True) 296 matcher.config.no_full_first_match().auto_memoize(full=True).trace_stack(True) 297 self.do_test(matcher.get_parse())

298

299 - def test_left_string_clear(self):

300 matcher = self.left() 301 matcher.config.clear().auto_memoize(full=True).trace_stack(True) 302 self.do_test(matcher.get_parse_string())

303

304 - def test_left_null_clear(self):

305 matcher = self.left() 306 matcher.config.clear().auto_memoize(full=True).trace_stack(True) 307 self.do_test(matcher.get_parse())

308

309 - def test_left_token_string_clear(self):

310 matcher = self.left_token() 311 matcher.config.clear().auto_memoize(full=True).lexer().trace_stack(True) 312 self.do_test(matcher.get_parse_string())

313

314 - def test_left_token_null_clear(self):

315 matcher = self.left_token() 316 matcher.config.clear().auto_memoize(full=True).lexer().trace_stack(True) 317 self.do_test(matcher.get_parse())

318

319 - def test_left_token_string_clear_content(self):

320 matcher = self.left_token(True) 321 matcher.config.clear().auto_memoize(full=True).lexer().trace_stack(True) 322 self.do_test(matcher.get_parse_string())

323

324 - def test_left_token_null_clear_content(self):

325 matcher = self.left_token(True) 326 matcher.config.clear().auto_memoize(full=True).lexer().trace_stack(True) 327 self.do_test(matcher.get_parse())

328 329 330 #class PerformanceTest(TestCase): 331 # 332 # def matcher(self): 333 # 334 # class VerbPhrase(Node): pass 335 # class DetPhrase(Node): pass 336 # class SimpleTp(Node): pass 337 # class TermPhrase(Node): pass 338 # class Sentence(Node): pass 339 # 340 # verb = Literals('knows', 'respects', 'loves') > 'verb' 341 # join = Literals('and', 'or') > 'join' 342 # proper_noun = Literals('helen', 'john', 'pat') > 'proper_noun' 343 # determiner = Literals('every', 'some') > 'determiner' 344 # noun = Literals('boy', 'girl', 'man', 'woman') > 'noun' 345 # 346 # verbphrase = Delayed() 347 # verbphrase += verb | (verbphrase // join // verbphrase) > VerbPhrase 348 # det_phrase = determiner // noun > DetPhrase 349 # simple_tp = proper_noun | det_phrase > SimpleTp 350 # termphrase = Delayed() 351 # termphrase += simple_tp | (termphrase // join // termphrase) > TermPhrase 352 # sentence = termphrase // verbphrase // termphrase & Eos() > Sentence 353 # 354 # return sentence 355 # 356 # def time_once(self, factory): 357 # matcher = factory().get_parse_all() 358 # start = time() 359 # assert len(list(matcher( 360 # 'every boy or some girl and helen and john or pat knows ' 361 # 'and respects or loves every boy or some girl and pat or ' 362 # 'john and helen'))) == 392 363 # return time() - start 364 # 365 # def repeat(self, count, factory): 366 # time = 0 367 # for _i in range(count): 368 # time += self.time_once(factory) 369 # return time 370 # 371 # def best_of(self, n, count, factory): 372 # times = [] 373 # for _i in range(n): 374 # times.append(self.repeat(count, factory)) 375 # return min(times) 376 # 377 # def test_timing(self): 378 # (n, count) = (3, 2) 379 # matcher = self.matcher() 380 # def default_factory(): 381 # matcher.config.clear() 382 # return matcher 383 # default = self.best_of(n, count, default_factory) 384 # def memo_factory(): 385 # matcher.config.clear().auto_memoize(full=True) 386 # return matcher 387 # memo = self.best_of(n, count, memo_factory) 388 # assert default > 10 * memo, (default, memo) 389

Source Code for Module lepl.matchers._test.memo