Mercurial > code > home > repos > homeauto
annotate service/mqtt_to_rdf/inference.py @ 1677:aa35ae7a1acc
add new bug test (no fix yet)
| author | drewp@bigasterisk.com |
|---|---|
| date | Wed, 22 Sep 2021 01:22:05 -0700 |
| parents | 4a15b4cd4600 |
| children | 2883da14847c |
| rev | line source |
|---|---|
| 1587 | 1 """ |
| 2 copied from reasoning 2021-08-29. probably same api. should | |
| 3 be able to lib/ this out | |
| 4 """ | |
|
1588
0757fafbfdab
WIP inferencer - partial var and function support
drewp@bigasterisk.com
parents:
1587
diff
changeset
|
5 import itertools |
| 1587 | 6 import logging |
| 1601 | 7 import time |
| 1594 | 8 from collections import defaultdict |
| 1626 | 9 from dataclasses import dataclass |
| 1673 | 10 from typing import Dict, Iterator, List, Optional, Union, cast |
|
1588
0757fafbfdab
WIP inferencer - partial var and function support
drewp@bigasterisk.com
parents:
1587
diff
changeset
|
11 |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
12 from prometheus_client import Histogram, Summary |
| 1673 | 13 from rdflib import Graph, Namespace |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
14 from rdflib.graph import ConjunctiveGraph |
| 1640 | 15 from rdflib.term import Node, URIRef, Variable |
| 1587 | 16 |
|
1638
0ba1625037ae
don't crash, just skip the rule if there's a BindingConflict (no clear test case yet)
drewp@bigasterisk.com
parents:
1637
diff
changeset
|
17 from candidate_binding import BindingConflict, CandidateBinding |
| 1673 | 18 from inference_types import (BindingUnknown, Inconsistent, RhsBnode, RuleOutBnode, RuleUnboundBnode, Triple, WorkingSetBnode) |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
19 from lhs_evaluation import functionsFor |
|
1650
2061df259224
move graphDump (on its way out, since reprs are getting better)
drewp@bigasterisk.com
parents:
1649
diff
changeset
|
20 from rdf_debug import graphDump |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
21 from stmt_chunk import AlignedRuleChunk, Chunk, ChunkedGraph, applyChunky |
| 1605 | 22 |
| 1587 | 23 log = logging.getLogger('infer') |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
24 odolog = logging.getLogger('infer.odo') # the "odometer" logic |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
25 ringlog = logging.getLogger('infer.ring') # for ChunkLooper |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
26 |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
27 INDENT = ' ' |
| 1587 | 28 |
| 1626 | 29 INFER_CALLS = Summary('inference_infer_calls', 'calls') |
| 30 INFER_GRAPH_SIZE = Histogram('inference_graph_size', 'statements', buckets=[2**x for x in range(2, 20, 2)]) | |
| 1587 | 31 |
| 32 ROOM = Namespace("http://projects.bigasterisk.com/room/") | |
| 33 LOG = Namespace('http://www.w3.org/2000/10/swap/log#') | |
| 34 MATH = Namespace('http://www.w3.org/2000/10/swap/math#') | |
| 35 | |
| 36 | |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
37 class NoOptions(ValueError): |
|
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
38 """ChunkLooper has no possibilites to add to the binding; the whole rule must therefore not apply""" |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
39 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
40 |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
41 _chunkLooperShortId = itertools.count() |
| 1632 | 42 |
| 43 | |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
44 @dataclass |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
45 class ChunkLooper: |
|
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
46 """given one LHS Chunk, iterate through the possible matches for it, |
| 1632 | 47 returning what bindings they would imply. Only distinct bindings are |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
48 returned. The bindings build on any `prev` ChunkLooper's results. |
| 1632 | 49 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
50 In the odometer metaphor used below, this is one of the rings. |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
51 |
| 1632 | 52 This iterator is restartable.""" |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
53 lhsChunk: Chunk |
|
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
54 prev: Optional['ChunkLooper'] |
|
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
55 workingSet: 'ChunkedGraph' |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
56 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
57 def __repr__(self): |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
58 return f'{self.__class__.__name__}{self._shortId}{"<pastEnd>" if self.pastEnd() else ""}' |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
59 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
60 def __post_init__(self): |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
61 self._shortId = next(_chunkLooperShortId) |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
62 self._alignedMatches = list(self.lhsChunk.ruleMatchesFrom(self.workingSet)) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
63 |
|
1668
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
64 # only ours- do not store prev, since it could change without us |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
65 self._current = CandidateBinding({}) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
66 self._pastEnd = False |
| 1667 | 67 self._seenBindings: List[CandidateBinding] = [] # combined bindings (up to our ring) that we've returned |
| 1648 | 68 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
69 if ringlog.isEnabledFor(logging.DEBUG): |
| 1667 | 70 ringlog.debug('') |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
71 ringlog.debug(f'{INDENT*6} introducing {self!r}({self.lhsChunk}, {self._alignedMatches=})') |
| 1648 | 72 |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
73 self.restart() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
74 |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
75 def _prevBindings(self) -> CandidateBinding: |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
76 if not self.prev or self.prev.pastEnd(): |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
77 return CandidateBinding({}) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
78 |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
79 return self.prev.currentBinding() |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
80 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
81 def advance(self): |
|
1668
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
82 """update _current to a new set of valid bindings we haven't seen (since |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
83 last restart), or go into pastEnd mode. Note that _current is just our |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
84 contribution, but returned valid bindings include all prev rings.""" |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
85 if self._pastEnd: |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
86 raise NotImplementedError('need restart') |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
87 ringlog.debug('') |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
88 augmentedWorkingSet: List[AlignedRuleChunk] = [] |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
89 if self.prev is None: |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
90 augmentedWorkingSet = self._alignedMatches |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
91 else: |
| 1673 | 92 augmentedWorkingSet = list(applyChunky(self.prev.currentBinding(), self._alignedMatches)) |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
93 |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
94 if self._advanceWithPlainMatches(augmentedWorkingSet): |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
95 ringlog.debug(f'{INDENT*6} <-- {self}.advance finished with plain matches') |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
96 return |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
97 |
| 1639 | 98 if self._advanceWithFunctions(): |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
99 ringlog.debug(f'{INDENT*6} <-- {self}.advance finished with function matches') |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
100 return |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
101 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
102 ringlog.debug(f'{INDENT*6} <-- {self}.advance had nothing and is now past end') |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
103 self._pastEnd = True |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
104 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
105 def _advanceWithPlainMatches(self, augmentedWorkingSet: List[AlignedRuleChunk]) -> bool: |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
106 ringlog.debug(f'{INDENT*7} {self} mines {len(augmentedWorkingSet)} matching augmented statements') |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
107 for s in augmentedWorkingSet: |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
108 ringlog.debug(f'{INDENT*8} {s}') |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
109 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
110 for aligned in augmentedWorkingSet: |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
111 try: |
|
1668
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
112 newBinding = aligned.newBindingIfMatched(self._prevBindings()) |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
113 except Inconsistent as exc: |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
114 ringlog.debug( |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
115 f'{INDENT*7} ChunkLooper{self._shortId} - {aligned} would be inconsistent with prev bindings ({exc})') |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
116 continue |
| 1632 | 117 |
|
1668
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
118 if self._testAndKeepNewBinding(newBinding): |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
119 return True |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
120 return False |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
121 |
| 1639 | 122 def _advanceWithFunctions(self) -> bool: |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
123 pred: Node = self.lhsChunk.predicate |
| 1640 | 124 if not isinstance(pred, URIRef): |
| 125 raise NotImplementedError | |
| 1632 | 126 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
127 ringlog.debug(f'{INDENT*6} advanceWithFunctions {pred!r}') |
| 1652 | 128 |
| 1637 | 129 for functionType in functionsFor(pred): |
| 1661 | 130 fn = functionType(self.lhsChunk) |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
131 ringlog.debug(f'{INDENT*7} ChunkLooper{self._shortId} advanceWithFunctions, {functionType=}') |
| 1652 | 132 |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
133 try: |
| 1652 | 134 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
135 newBinding = fn.bind(self._prevBindings()) |
| 1637 | 136 except BindingUnknown: |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
137 pass |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
138 else: |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
139 if newBinding is not None: |
|
1668
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
140 if self._testAndKeepNewBinding(newBinding): |
| 1637 | 141 return True |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
142 |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
143 return False |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
144 |
|
1668
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
145 def _testAndKeepNewBinding(self, newBinding): |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
146 fullBinding: CandidateBinding = self._prevBindings().copy() |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
147 fullBinding.addNewBindings(newBinding) |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
148 isNew = fullBinding not in self._seenBindings |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
149 ringlog.debug(f'{INDENT*7} {self} considering {newBinding=} to make {fullBinding}. {isNew=}') |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
150 if isNew: |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
151 self._seenBindings.append(fullBinding.copy()) |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
152 self._current = newBinding |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
153 return True |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
154 return False |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
155 |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
156 def _boundOperands(self, operands) -> List[Node]: |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
157 pb: CandidateBinding = self._prevBindings() |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
158 |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
159 boundOperands: List[Node] = [] |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
160 for op in operands: |
| 1673 | 161 if isinstance(op, (Variable, RuleUnboundBnode)): |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
162 boundOperands.append(pb.applyTerm(op)) |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
163 else: |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
164 boundOperands.append(op) |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
165 return boundOperands |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
166 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
167 def currentBinding(self) -> CandidateBinding: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
168 if self.pastEnd(): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
169 raise NotImplementedError() |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
170 together = self._prevBindings().copy() |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
171 together.addNewBindings(self._current) |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
172 return together |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
173 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
174 def pastEnd(self) -> bool: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
175 return self._pastEnd |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
176 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
177 def restart(self): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
178 self._pastEnd = False |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
179 self._seenBindings = [] |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
180 self.advance() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
181 if self.pastEnd(): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
182 raise NoOptions() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
183 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
184 |
| 1594 | 185 @dataclass |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
186 class Lhs: |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
187 graph: ChunkedGraph # our full LHS graph, as input. See below for the statements partitioned into groups. |
| 1594 | 188 |
| 189 def __post_init__(self): | |
| 1648 | 190 |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
191 self.myPreds = self.graph.allPredicatesExceptFunctions() |
| 1602 | 192 |
|
1609
34f2817320cc
new tests for a smaller part of the inner loop
drewp@bigasterisk.com
parents:
1608
diff
changeset
|
193 def __repr__(self): |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
194 return f"Lhs({self.graph!r})" |
|
1609
34f2817320cc
new tests for a smaller part of the inner loop
drewp@bigasterisk.com
parents:
1608
diff
changeset
|
195 |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
196 def findCandidateBindings(self, knownTrue: ChunkedGraph, stats, ruleStatementsIterationLimit) -> Iterator['BoundLhs']: |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
197 """distinct bindings that fit the LHS of a rule, using statements from |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
198 workingSet and functions from LHS""" |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
199 if not self.graph: |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
200 # special case- no LHS! |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
201 yield BoundLhs(self, CandidateBinding({})) |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
202 return |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
203 |
| 1640 | 204 if self._checkPredicateCounts(knownTrue): |
| 205 stats['_checkPredicateCountsCulls'] += 1 | |
| 206 return | |
| 207 | |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
208 if not all(ch in knownTrue for ch in self.graph.staticChunks): |
| 1648 | 209 stats['staticStmtCulls'] += 1 |
| 210 return | |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
211 # After this point we don't need to consider self.graph.staticChunks. |
| 1648 | 212 |
|
1656
d8e5b2232474
bugfix for simple (no arg lists) functions not evaluating
drewp@bigasterisk.com
parents:
1653
diff
changeset
|
213 if not self.graph.patternChunks and not self.graph.chunksUsedByFuncs: |
| 1648 | 214 # static only |
| 215 yield BoundLhs(self, CandidateBinding({})) | |
| 216 return | |
| 217 | |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
218 log.debug('') |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
219 try: |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
220 chunkStack = self._assembleRings(knownTrue, stats) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
221 except NoOptions: |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
222 ringlog.debug(f'{INDENT*5} start up with no options; 0 bindings') |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
223 return |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
224 self._debugChunkStack('time to spin: initial odometer is', chunkStack) |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
225 self._assertAllRingsAreValid(chunkStack) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
226 |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
227 lastRing = chunkStack[-1] |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
228 iterCount = 0 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
229 while True: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
230 iterCount += 1 |
| 1648 | 231 if iterCount > ruleStatementsIterationLimit: |
| 232 raise ValueError('rule too complex') | |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
233 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
234 log.debug(f'{INDENT*4} vv findCandBindings iteration {iterCount}') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
235 |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
236 yield BoundLhs(self, lastRing.currentBinding()) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
237 |
| 1667 | 238 # self._debugChunkStack('odometer', chunkStack) |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
239 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
240 done = self._advanceTheStack(chunkStack) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
241 |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
242 self._debugChunkStack(f'odometer after ({done=})', chunkStack) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
243 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
244 log.debug(f'{INDENT*4} ^^ findCandBindings iteration done') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
245 if done: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
246 break |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
247 |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
248 def _debugChunkStack(self, label: str, chunkStack: List[ChunkLooper]): |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
249 odolog.debug(f'{INDENT*4} {label}:') |
| 1652 | 250 for i, l in enumerate(chunkStack): |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
251 odolog.debug(f'{INDENT*5} [{i}] {l} curbind={l.currentBinding() if not l.pastEnd() else "<end>"}') |
| 1632 | 252 |
| 1640 | 253 def _checkPredicateCounts(self, knownTrue): |
| 254 """raise NoOptions quickly in some cases""" | |
| 1648 | 255 |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
256 if self.graph.noPredicatesAppear(self.myPreds): |
| 1652 | 257 log.debug(f'{INDENT*3} checkPredicateCounts does cull because not all {self.myPreds=} are in knownTrue') |
| 1640 | 258 return True |
| 1652 | 259 log.debug(f'{INDENT*3} checkPredicateCounts does not cull because all {self.myPreds=} are in knownTrue') |
| 1640 | 260 return False |
| 261 | |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
262 def _assembleRings(self, knownTrue: ChunkedGraph, stats) -> List[ChunkLooper]: |
|
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
263 """make ChunkLooper for each stmt in our LHS graph, but do it in a way that they all |
| 1652 | 264 start out valid (or else raise NoOptions). static chunks have already been confirmed.""" |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
265 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
266 log.debug(f'{INDENT*4} stats={dict(stats)}') |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
267 odolog.debug(f'{INDENT*3} build new ChunkLooper stack') |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
268 chunks = list(self.graph.patternChunks.union(self.graph.chunksUsedByFuncs)) |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
269 chunks.sort(key=None) |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
270 odolog.info(f' {INDENT*3} taking permutations of {len(chunks)=}') |
| 1653 | 271 for i, perm in enumerate(itertools.permutations(chunks)): |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
272 looperRings: List[ChunkLooper] = [] |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
273 prev: Optional[ChunkLooper] = None |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
274 if odolog.isEnabledFor(logging.DEBUG): |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
275 odolog.debug(f'{INDENT*4} [perm {i}] try rule chunks in this order: {" THEN ".join(repr(p) for p in perm)}') |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
276 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
277 for ruleChunk in perm: |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
278 try: |
| 1661 | 279 # These are getting rebuilt a lot which takes time. It would |
| 280 # be nice if they could accept a changing `prev` order | |
| 281 # (which might already be ok). | |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
282 looper = ChunkLooper(ruleChunk, prev, knownTrue) |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
283 except NoOptions: |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
284 odolog.debug(f'{INDENT*5} permutation didnt work, try another') |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
285 break |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
286 looperRings.append(looper) |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
287 prev = looperRings[-1] |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
288 else: |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
289 # bug: At this point we've only shown that these are valid |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
290 # starting rings. The rules might be tricky enough that this |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
291 # permutation won't get us to the solution. |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
292 return looperRings |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
293 if i > 50000: |
| 1652 | 294 raise NotImplementedError(f'trying too many permutations {len(chunks)=}') |
| 1648 | 295 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
296 odolog.debug(f'{INDENT*5} no perms worked- rule cannot match anything') |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
297 raise NoOptions() |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
298 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
299 def _advanceTheStack(self, looperRings: List[ChunkLooper]) -> bool: |
|
1668
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
300 carry = True # last elem always must advance |
|
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
301 for i, ring in reversed(list(enumerate(looperRings))): |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
302 # unlike normal odometer, advancing any earlier ring could invalidate later ones |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
303 if carry: |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
304 odolog.debug(f'{INDENT*4} advanceAll [{i}] {ring} carry/advance') |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
305 ring.advance() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
306 carry = False |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
307 if ring.pastEnd(): |
|
1668
89e53cb8a01c
fix some harder tests. Mostly, _advanceTheStack needed to spin the odometer rings starting from the other side, to get all the right combos
drewp@bigasterisk.com
parents:
1667
diff
changeset
|
308 if ring is looperRings[0]: |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
309 allRingsDone = [r.pastEnd() for r in looperRings] |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
310 odolog.debug(f'{INDENT*4} advanceAll [{i}] {ring} says we done {allRingsDone=}') |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
311 return True |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
312 odolog.debug(f'{INDENT*4} advanceAll [{i}] {ring} restart') |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
313 ring.restart() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
314 carry = True |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
315 return False |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
316 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
317 def _assertAllRingsAreValid(self, looperRings): |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
318 if any(ring.pastEnd() for ring in looperRings): # this is an unexpected debug assertion |
|
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
319 odolog.warning(f'{INDENT*4} some rings started at pastEnd {looperRings}') |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
320 raise NoOptions() |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
321 |
|
1592
d7b66234064b
pure reordering of funcs to make the next diffs smaller
drewp@bigasterisk.com
parents:
1591
diff
changeset
|
322 |
|
1622
38bd8ef9ef67
add CandidateTermMatches, unused so far
drewp@bigasterisk.com
parents:
1621
diff
changeset
|
323 @dataclass |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
324 class BoundLhs: |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
325 lhs: Lhs |
| 1610 | 326 binding: CandidateBinding |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
327 |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
328 |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
329 @dataclass |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
330 class Rule: |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
331 lhsGraph: Graph |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
332 rhsGraph: Graph |
| 1632 | 333 |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
334 def __post_init__(self): |
| 1673 | 335 self.lhs = Lhs(ChunkedGraph(self.lhsGraph, RuleUnboundBnode, functionsFor)) |
| 336 | |
| 337 self.maps = {} | |
| 338 | |
| 339 self.rhsGraphConvert: List[Triple] = [] | |
| 340 for s, p, o in self.rhsGraph: | |
| 341 from rdflib import BNode | |
| 342 if isinstance(s, BNode): | |
| 343 s = RhsBnode(s) | |
| 344 if isinstance(p, BNode): | |
| 345 p = RhsBnode(p) | |
| 346 if isinstance(o, BNode): | |
| 347 o = RhsBnode(o) | |
| 348 self.rhsGraphConvert.append((s, p, o)) | |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
349 |
| 1648 | 350 def applyRule(self, workingSet: Graph, implied: Graph, stats: Dict, ruleStatementsIterationLimit): |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
351 # this does not change for the current applyRule call. The rule will be |
|
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
352 # tried again in an outer loop, in case it can produce more. |
| 1673 | 353 workingSetChunked = ChunkedGraph(workingSet, WorkingSetBnode, functionsFor) |
|
1651
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
354 |
|
20474ad4968e
WIP - functions are broken as i move most layers to work in Chunks not Triples
drewp@bigasterisk.com
parents:
1650
diff
changeset
|
355 for bound in self.lhs.findCandidateBindings(workingSetChunked, stats, ruleStatementsIterationLimit): |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
356 log.debug(f'{INDENT*5} +rule has a working binding: {bound}') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
357 |
| 1673 | 358 newStmts = self.generateImpliedFromRhs(bound.binding) |
|
1612
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
359 |
| 1673 | 360 for newStmt in newStmts: |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
361 # log.debug(f'{INDENT*6} adding {newStmt=}') |
|
1612
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
362 workingSet.add(newStmt) |
|
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
363 implied.add(newStmt) |
|
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
364 |
| 1673 | 365 def generateImpliedFromRhs(self, binding: CandidateBinding) -> List[Triple]: |
| 366 | |
| 367 out: List[Triple] = [] | |
| 368 | |
| 369 # Each time the RHS is used (in a rule firing), its own BNodes (which | |
| 370 # are subtype RhsBnode) need to be turned into distinct ones. Note that | |
| 371 # bnodes that come from the working set should not be remapped. | |
| 372 rhsBnodeMap: Dict[RhsBnode, WorkingSetBnode] = {} | |
| 373 | |
| 374 # but, the iteration loop could come back with the same bindings again | |
| 375 key = binding.key() | |
| 376 rhsBnodeMap = self.maps.setdefault(key, {}) | |
| 377 | |
| 378 for stmt in binding.apply(self.rhsGraphConvert): | |
| 379 | |
| 380 outStmt: List[Node] = [] | |
| 381 | |
| 382 for t in stmt: | |
| 383 if isinstance(t, RhsBnode): | |
| 384 if t not in rhsBnodeMap: | |
| 385 rhsBnodeMap[t] = WorkingSetBnode() | |
| 386 t = rhsBnodeMap[t] | |
| 387 | |
| 388 outStmt.append(t) | |
| 389 | |
| 390 log.debug(f'{INDENT*6} rhs stmt {stmt} became {outStmt}') | |
| 391 out.append((outStmt[0], outStmt[1], outStmt[2])) | |
| 392 | |
| 393 return out | |
| 394 | |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
395 |
| 1648 | 396 @dataclass |
| 1587 | 397 class Inference: |
| 1673 | 398 rulesIterationLimit = 4 |
|
1664
1a7c1261302c
logic fix- some bindings were being returned 2+; some 0 times
drewp@bigasterisk.com
parents:
1661
diff
changeset
|
399 ruleStatementsIterationLimit = 5000 |
| 1587 | 400 |
| 401 def __init__(self) -> None: | |
| 1648 | 402 self.rules: List[Rule] = [] |
| 403 self._nonRuleStmts: List[Triple] = [] | |
| 1587 | 404 |
| 405 def setRules(self, g: ConjunctiveGraph): | |
| 1648 | 406 self.rules = [] |
| 407 self._nonRuleStmts = [] | |
|
1599
abbf0eb0e640
fix a bug with a slightly moer complicated set of rules
drewp@bigasterisk.com
parents:
1598
diff
changeset
|
408 for stmt in g: |
|
abbf0eb0e640
fix a bug with a slightly moer complicated set of rules
drewp@bigasterisk.com
parents:
1598
diff
changeset
|
409 if stmt[1] == LOG['implies']: |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
410 self.rules.append(Rule(stmt[0], stmt[2])) |
|
1649
bb5d2b5370ac
add nonRuleStatments to Inference api. there's already a test in an eariler commit
drewp@bigasterisk.com
parents:
1648
diff
changeset
|
411 else: |
|
bb5d2b5370ac
add nonRuleStatments to Inference api. there's already a test in an eariler commit
drewp@bigasterisk.com
parents:
1648
diff
changeset
|
412 self._nonRuleStmts.append(stmt) |
|
bb5d2b5370ac
add nonRuleStatments to Inference api. there's already a test in an eariler commit
drewp@bigasterisk.com
parents:
1648
diff
changeset
|
413 |
|
bb5d2b5370ac
add nonRuleStatments to Inference api. there's already a test in an eariler commit
drewp@bigasterisk.com
parents:
1648
diff
changeset
|
414 def nonRuleStatements(self) -> List[Triple]: |
|
bb5d2b5370ac
add nonRuleStatments to Inference api. there's already a test in an eariler commit
drewp@bigasterisk.com
parents:
1648
diff
changeset
|
415 return self._nonRuleStmts |
| 1587 | 416 |
| 1601 | 417 @INFER_CALLS.time() |
| 1587 | 418 def infer(self, graph: Graph): |
| 419 """ | |
| 420 returns new graph of inferred statements. | |
| 421 """ | |
| 1626 | 422 n = graph.__len__() |
| 423 INFER_GRAPH_SIZE.observe(n) | |
| 424 log.info(f'{INDENT*0} Begin inference of graph len={n} with rules len={len(self.rules)}:') | |
| 1601 | 425 startTime = time.time() |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
426 stats: Dict[str, Union[int, float]] = defaultdict(lambda: 0) |
|
1649
bb5d2b5370ac
add nonRuleStatments to Inference api. there's already a test in an eariler commit
drewp@bigasterisk.com
parents:
1648
diff
changeset
|
427 |
|
1589
5c1055be3c36
WIP more debugging, working towards bnode-matching support
drewp@bigasterisk.com
parents:
1588
diff
changeset
|
428 # everything that is true: the input graph, plus every rule conclusion we can make |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
429 workingSet = Graph() |
|
1649
bb5d2b5370ac
add nonRuleStatments to Inference api. there's already a test in an eariler commit
drewp@bigasterisk.com
parents:
1648
diff
changeset
|
430 workingSet += self._nonRuleStmts |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
431 workingSet += graph |
| 1587 | 432 |
| 1594 | 433 # just the statements that came from RHS's of rules that fired. |
| 1587 | 434 implied = ConjunctiveGraph() |
| 435 | |
| 1648 | 436 rulesIterations = 0 |
| 1587 | 437 delta = 1 |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
438 stats['initWorkingSet'] = cast(int, workingSet.__len__()) |
|
1674
4a15b4cd4600
don't jsut quietly succeed if iteration limit is reached
drewp@bigasterisk.com
parents:
1673
diff
changeset
|
439 while delta > 0: |
| 1620 | 440 log.debug('') |
| 1648 | 441 log.info(f'{INDENT*1}*iteration {rulesIterations}') |
| 442 | |
| 1587 | 443 delta = -len(implied) |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
444 self._iterateAllRules(workingSet, implied, stats) |
| 1587 | 445 delta += len(implied) |
| 1648 | 446 rulesIterations += 1 |
| 1597 | 447 log.info(f'{INDENT*2} this inference iteration added {delta} more implied stmts') |
|
1674
4a15b4cd4600
don't jsut quietly succeed if iteration limit is reached
drewp@bigasterisk.com
parents:
1673
diff
changeset
|
448 if rulesIterations >= self.rulesIterationLimit: |
|
4a15b4cd4600
don't jsut quietly succeed if iteration limit is reached
drewp@bigasterisk.com
parents:
1673
diff
changeset
|
449 raise ValueError(f"rule too complex after {rulesIterations=}") |
| 1648 | 450 stats['iterations'] = rulesIterations |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
451 stats['timeSpent'] = round(time.time() - startTime, 3) |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
452 stats['impliedStmts'] = len(implied) |
| 1648 | 453 log.info(f'{INDENT*0} Inference done {dict(stats)}.') |
| 454 log.debug('Implied:') | |
| 455 log.debug(graphDump(implied)) | |
| 1587 | 456 return implied |
| 457 | |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
458 def _iterateAllRules(self, workingSet: Graph, implied: Graph, stats): |
|
1612
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
459 for i, rule in enumerate(self.rules): |
|
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
460 self._logRuleApplicationHeader(workingSet, i, rule) |
| 1648 | 461 rule.applyRule(workingSet, implied, stats, self.ruleStatementsIterationLimit) |
| 1587 | 462 |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
463 def _logRuleApplicationHeader(self, workingSet, i, r: Rule): |
| 1594 | 464 if not log.isEnabledFor(logging.DEBUG): |
| 465 return | |
| 466 | |
| 467 log.debug('') | |
| 468 log.debug(f'{INDENT*2} workingSet:') | |
| 1648 | 469 # for j, stmt in enumerate(sorted(workingSet)): |
| 470 # log.debug(f'{INDENT*3} ({j}) {stmt}') | |
| 471 log.debug(f'{INDENT*3} {graphDump(workingSet, oneLine=False)}') | |
| 1594 | 472 |
| 473 log.debug('') | |
| 474 log.debug(f'{INDENT*2}-applying rule {i}') | |
| 1632 | 475 log.debug(f'{INDENT*3} rule def lhs:') |
| 1677 | 476 for stmt in sorted(r.lhs.graph.allChunks()): |
| 1632 | 477 log.debug(f'{INDENT*4} {stmt}') |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
478 log.debug(f'{INDENT*3} rule def rhs: {graphDump(r.rhsGraph)}') |