Mercurial > code > home > repos > homeauto

comparison service/mqtt_to_rdf/inference.py @ 1692:2883da14847c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
debugging and cleanup, as i looked for a bug
author drewp@bigasterisk.com
date Sat, 25 Sep 2021 22:20:00 -0700
parents aa35ae7a1acc
children 0455a1e18e4f
comparison
equal deleted inserted replaced
1691:37710d28890b 1692:2883da14847c
5 import itertools 5 import itertools
6 import logging 6 import logging
7 import time 7 import time
8 from collections import defaultdict 8 from collections import defaultdict
9 from dataclasses import dataclass 9 from dataclasses import dataclass
10 from typing import Dict, Iterator, List, Optional, Union, cast 10 from typing import Dict, Iterator, List, Optional, Tuple, Union, cast
11 11
12 from prometheus_client import Histogram, Summary 12 from prometheus_client import Histogram, Summary
13 from rdflib import Graph, Namespace 13 from rdflib import Graph, Namespace
14 from rdflib.graph import ConjunctiveGraph 14 from rdflib.graph import ConjunctiveGraph
15 from rdflib.term import Node, URIRef, Variable 15 from rdflib.term import Node, URIRef, Variable
129 for functionType in functionsFor(pred): 129 for functionType in functionsFor(pred):
130 fn = functionType(self.lhsChunk) 130 fn = functionType(self.lhsChunk)
131 ringlog.debug(f'{INDENT*7} ChunkLooper{self._shortId} advanceWithFunctions, {functionType=}') 131 ringlog.debug(f'{INDENT*7} ChunkLooper{self._shortId} advanceWithFunctions, {functionType=}')
132 132
133 try: 133 try:
134 134 log.debug(f'fn.bind {self._prevBindings()} ...')
135 #fullBinding = self._prevBindings().copy()
135 newBinding = fn.bind(self._prevBindings()) 136 newBinding = fn.bind(self._prevBindings())
137 log.debug(f'...makes {newBinding=}')
136 except BindingUnknown: 138 except BindingUnknown:
137 pass 139 pass
138 else: 140 else:
139 if newBinding is not None: 141 if newBinding is not None:
140 if self._testAndKeepNewBinding(newBinding): 142 if self._testAndKeepNewBinding(newBinding):
141 return True 143 return True
142 144
143 return False 145 return False
144 146
145 def _testAndKeepNewBinding(self, newBinding): 147 def _testAndKeepNewBinding(self, newBinding: CandidateBinding):
146 fullBinding: CandidateBinding = self._prevBindings().copy() 148 fullBinding: CandidateBinding = self._prevBindings().copy()
147 fullBinding.addNewBindings(newBinding) 149 fullBinding.addNewBindings(newBinding)
148 isNew = fullBinding not in self._seenBindings 150 isNew = fullBinding not in self._seenBindings
149 ringlog.debug(f'{INDENT*7} {self} considering {newBinding=} to make {fullBinding}. {isNew=}') 151 ringlog.debug(f'{INDENT*7} {self} considering {newBinding=} to make {fullBinding}. {isNew=}')
150 if isNew: 152 if isNew:
219 try: 221 try:
220 chunkStack = self._assembleRings(knownTrue, stats) 222 chunkStack = self._assembleRings(knownTrue, stats)
221 except NoOptions: 223 except NoOptions:
222 ringlog.debug(f'{INDENT*5} start up with no options; 0 bindings') 224 ringlog.debug(f'{INDENT*5} start up with no options; 0 bindings')
223 return 225 return
226 log.debug('')
227 log.debug('')
224 self._debugChunkStack('time to spin: initial odometer is', chunkStack) 228 self._debugChunkStack('time to spin: initial odometer is', chunkStack)
225 self._assertAllRingsAreValid(chunkStack) 229 self._assertAllRingsAreValid(chunkStack)
226 230
227 lastRing = chunkStack[-1] 231 lastRing = chunkStack[-1]
228 iterCount = 0 232 iterCount = 0
266 log.debug(f'{INDENT*4} stats={dict(stats)}') 270 log.debug(f'{INDENT*4} stats={dict(stats)}')
267 odolog.debug(f'{INDENT*3} build new ChunkLooper stack') 271 odolog.debug(f'{INDENT*3} build new ChunkLooper stack')
268 chunks = list(self.graph.patternChunks.union(self.graph.chunksUsedByFuncs)) 272 chunks = list(self.graph.patternChunks.union(self.graph.chunksUsedByFuncs))
269 chunks.sort(key=None) 273 chunks.sort(key=None)
270 odolog.info(f' {INDENT*3} taking permutations of {len(chunks)=}') 274 odolog.info(f' {INDENT*3} taking permutations of {len(chunks)=}')
271 for i, perm in enumerate(itertools.permutations(chunks)): 275
276 permsTried = 0
277
278 for perm in self._partitionedGraphPermutations():
272 looperRings: List[ChunkLooper] = [] 279 looperRings: List[ChunkLooper] = []
273 prev: Optional[ChunkLooper] = None 280 prev: Optional[ChunkLooper] = None
274 if odolog.isEnabledFor(logging.DEBUG): 281 if odolog.isEnabledFor(logging.DEBUG):
275 odolog.debug(f'{INDENT*4} [perm {i}] try rule chunks in this order: {" THEN ".join(repr(p) for p in perm)}') 282 odolog.debug(
283 f'{INDENT*4} [perm {permsTried}] try rule chunks in this order: {" THEN ".join(repr(p) for p in perm)}')
276 284
277 for ruleChunk in perm: 285 for ruleChunk in perm:
278 try: 286 try:
279 # These are getting rebuilt a lot which takes time. It would 287 # These are getting rebuilt a lot which takes time. It would
280 # be nice if they could accept a changing `prev` order 288 # be nice if they could accept a changing `prev` order
288 else: 296 else:
289 # bug: At this point we've only shown that these are valid 297 # bug: At this point we've only shown that these are valid
290 # starting rings. The rules might be tricky enough that this 298 # starting rings. The rules might be tricky enough that this
291 # permutation won't get us to the solution. 299 # permutation won't get us to the solution.
292 return looperRings 300 return looperRings
293 if i > 50000: 301 if permsTried > 50000:
294 raise NotImplementedError(f'trying too many permutations {len(chunks)=}') 302 raise NotImplementedError(f'trying too many permutations {len(chunks)=}')
303 permsTried += 1
295 304
296 odolog.debug(f'{INDENT*5} no perms worked- rule cannot match anything') 305 odolog.debug(f'{INDENT*5} no perms worked- rule cannot match anything')
297 raise NoOptions() 306 raise NoOptions()
307
308 def _unpartitionedGraphPermutations(self) -> Iterator[Tuple[Chunk, ...]]:
309 for perm in itertools.permutations(sorted(list(self.graph.patternChunks.union(self.graph.chunksUsedByFuncs)))):
310 yield perm
311
312 def _partitionedGraphPermutations(self) -> Iterator[Tuple[Chunk, ...]]:
313 """always puts function chunks after pattern chunks
314
315 (and, if we cared, static chunks could go before that. Currently they're
316 culled out elsewhere, but that's done as a special case)
317 """
318 tupleOfNoChunks: Tuple[Chunk, ...] = ()
319 pats = sorted(self.graph.patternChunks)
320 funcs = sorted(self.graph.chunksUsedByFuncs)
321 for patternPart in itertools.permutations(pats) if pats else [tupleOfNoChunks]:
322 for funcPart in itertools.permutations(funcs) if funcs else [tupleOfNoChunks]:
323 perm = patternPart + funcPart
324 yield perm
298 325
299 def _advanceTheStack(self, looperRings: List[ChunkLooper]) -> bool: 326 def _advanceTheStack(self, looperRings: List[ChunkLooper]) -> bool:
300 carry = True # last elem always must advance 327 carry = True # last elem always must advance
301 for i, ring in reversed(list(enumerate(looperRings))): 328 for i, ring in reversed(list(enumerate(looperRings))):
302 # unlike normal odometer, advancing any earlier ring could invalidate later ones 329 # unlike normal odometer, advancing any earlier ring could invalidate later ones
303 if carry: 330 if carry:
304 odolog.debug(f'{INDENT*4} advanceAll [{i}] {ring} carry/advance') 331 odolog.debug(f'{INDENT*4} advanceAll [ring={i}] {ring} carry/advance')
305 ring.advance() 332 ring.advance()
306 carry = False 333 carry = False
307 if ring.pastEnd(): 334 if ring.pastEnd():
308 if ring is looperRings[0]: 335 if ring is looperRings[0]:
309 allRingsDone = [r.pastEnd() for r in looperRings] 336 allRingsDone = [r.pastEnd() for r in looperRings]
310 odolog.debug(f'{INDENT*4} advanceAll [{i}] {ring} says we done {allRingsDone=}') 337 odolog.debug(f'{INDENT*5} advanceAll [ring={i}] {ring} says we done {allRingsDone=}')
311 return True 338 return True
312 odolog.debug(f'{INDENT*4} advanceAll [{i}] {ring} restart') 339 odolog.debug(f'{INDENT*5} advanceAll [ring={i}] {ring} restart')
313 ring.restart() 340 ring.restart()
314 carry = True 341 carry = True
315 return False 342 return False
316 343
317 def _assertAllRingsAreValid(self, looperRings): 344 def _assertAllRingsAreValid(self, looperRings):