Mercurial > code > home > repos > homeauto
annotate service/mqtt_to_rdf/inference.py @ 1648:3059f31b2dfa
more performance work
| author | drewp@bigasterisk.com |
|---|---|
| date | Fri, 17 Sep 2021 11:10:18 -0700 |
| parents | 5403c6343fa4 |
| children | bb5d2b5370ac |
| rev | line source |
|---|---|
| 1587 | 1 """ |
| 2 copied from reasoning 2021-08-29. probably same api. should | |
| 3 be able to lib/ this out | |
| 4 """ | |
|
1588
0757fafbfdab
WIP inferencer - partial var and function support
drewp@bigasterisk.com
parents:
1587
diff
changeset
|
5 import itertools |
| 1587 | 6 import logging |
| 1601 | 7 import time |
| 1594 | 8 from collections import defaultdict |
| 1626 | 9 from dataclasses import dataclass |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
10 from typing import (Dict, Iterator, List, Optional, Sequence, Set, Tuple, Union, cast) |
|
1588
0757fafbfdab
WIP inferencer - partial var and function support
drewp@bigasterisk.com
parents:
1587
diff
changeset
|
11 |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
12 from prometheus_client import Histogram, Summary |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
13 from rdflib import RDF, BNode, Graph, Namespace |
|
1589
5c1055be3c36
WIP more debugging, working towards bnode-matching support
drewp@bigasterisk.com
parents:
1588
diff
changeset
|
14 from rdflib.graph import ConjunctiveGraph, ReadOnlyGraphAggregate |
| 1640 | 15 from rdflib.term import Node, URIRef, Variable |
| 1587 | 16 |
|
1638
0ba1625037ae
don't crash, just skip the rule if there's a BindingConflict (no clear test case yet)
drewp@bigasterisk.com
parents:
1637
diff
changeset
|
17 from candidate_binding import BindingConflict, CandidateBinding |
| 1637 | 18 from inference_types import BindingUnknown, ReadOnlyWorkingSet, Triple |
| 1640 | 19 from lhs_evaluation import functionsFor, lhsStmtsUsedByFuncs, rulePredicates |
| 1605 | 20 |
| 1587 | 21 log = logging.getLogger('infer') |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
22 INDENT = ' ' |
| 1587 | 23 |
| 1626 | 24 INFER_CALLS = Summary('inference_infer_calls', 'calls') |
| 25 INFER_GRAPH_SIZE = Histogram('inference_graph_size', 'statements', buckets=[2**x for x in range(2, 20, 2)]) | |
| 1587 | 26 |
| 27 ROOM = Namespace("http://projects.bigasterisk.com/room/") | |
| 28 LOG = Namespace('http://www.w3.org/2000/10/swap/log#') | |
| 29 MATH = Namespace('http://www.w3.org/2000/10/swap/math#') | |
| 30 | |
| 31 | |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
32 def stmtTemplate(stmt: Triple) -> Tuple[Optional[Node], Optional[Node], Optional[Node]]: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
33 return ( |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
34 None if isinstance(stmt[0], (Variable, BNode)) else stmt[0], |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
35 None if isinstance(stmt[1], (Variable, BNode)) else stmt[1], |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
36 None if isinstance(stmt[2], (Variable, BNode)) else stmt[2], |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
37 ) |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
38 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
39 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
40 class NoOptions(ValueError): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
41 """stmtlooper has no possibilites to add to the binding; the whole rule must therefore not apply""" |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
42 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
43 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
44 class Inconsistent(ValueError): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
45 """adding this stmt would be inconsistent with an existing binding""" |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
46 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
47 |
| 1632 | 48 _stmtLooperShortId = itertools.count() |
| 49 | |
| 50 | |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
51 @dataclass |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
52 class StmtLooper: |
| 1632 | 53 """given one LHS stmt, iterate through the possible matches for it, |
| 54 returning what bindings they would imply. Only distinct bindings are | |
| 55 returned. The bindings build on any `prev` StmtLooper's results. | |
| 56 | |
| 57 This iterator is restartable.""" | |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
58 lhsStmt: Triple |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
59 prev: Optional['StmtLooper'] |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
60 workingSet: ReadOnlyWorkingSet |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
61 parent: 'Lhs' # just for lhs.graph, really |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
62 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
63 def __repr__(self): |
| 1648 | 64 return f'StmtLooper{self._shortId}{"<pastEnd>" if self.pastEnd() else ""})' |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
65 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
66 def __post_init__(self): |
| 1632 | 67 self._shortId = next(_stmtLooperShortId) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
68 self._myWorkingSetMatches = self._myMatches(self.workingSet) |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
69 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
70 self._current = CandidateBinding({}) |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
71 self._pastEnd = False |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
72 self._seenBindings: List[CandidateBinding] = [] |
| 1648 | 73 |
| 74 log.debug(f'introducing {self!r}({graphDump([self.lhsStmt])})') | |
| 75 | |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
76 self.restart() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
77 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
78 def _myMatches(self, g: Graph) -> List[Triple]: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
79 template = stmtTemplate(self.lhsStmt) |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
80 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
81 stmts = sorted(cast(Iterator[Triple], list(g.triples(template)))) |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
82 # plus new lhs possibilties... |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
83 # log.debug(f'{INDENT*6} {self} find {len(stmts)=} in {len(self.workingSet)=}') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
84 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
85 return stmts |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
86 |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
87 def _prevBindings(self) -> CandidateBinding: |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
88 if not self.prev or self.prev.pastEnd(): |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
89 return CandidateBinding({}) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
90 |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
91 return self.prev.currentBinding() |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
92 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
93 def advance(self): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
94 """update to a new set of bindings we haven't seen (since last restart), or go into pastEnd mode""" |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
95 if self._pastEnd: |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
96 raise NotImplementedError('need restart') |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
97 log.debug('') |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
98 augmentedWorkingSet: Sequence[Triple] = [] |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
99 if self.prev is None: |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
100 augmentedWorkingSet = self._myWorkingSetMatches |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
101 else: |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
102 augmentedWorkingSet = list(self.prev.currentBinding().apply(self._myWorkingSetMatches, |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
103 returnBoundStatementsOnly=False)) |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
104 |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
105 log.debug(f'{INDENT*6} {self}.advance has {augmentedWorkingSet=}') |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
106 |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
107 if self._advanceWithPlainMatches(augmentedWorkingSet): |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
108 return |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
109 |
| 1639 | 110 if self._advanceWithFunctions(): |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
111 return |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
112 |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
113 log.debug(f'{INDENT*6} {self} is past end') |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
114 self._pastEnd = True |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
115 |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
116 def _advanceWithPlainMatches(self, augmentedWorkingSet: Sequence[Triple]) -> bool: |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
117 log.debug(f'{INDENT*7} {self} mines {len(augmentedWorkingSet)} matching augmented statements') |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
118 for s in augmentedWorkingSet: |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
119 log.debug(f'{INDENT*7} {s}') |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
120 |
| 1637 | 121 for stmt in augmentedWorkingSet: |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
122 try: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
123 outBinding = self._totalBindingIfThisStmtWereTrue(stmt) |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
124 except Inconsistent: |
| 1648 | 125 log.debug(f'{INDENT*7} StmtLooper{self._shortId} - {stmt} would be inconsistent with prev bindings') |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
126 continue |
| 1632 | 127 |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
128 log.debug(f'{INDENT*7} {outBinding=} {self._seenBindings=}') |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
129 if outBinding not in self._seenBindings: |
|
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
130 self._seenBindings.append(outBinding.copy()) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
131 self._current = outBinding |
| 1632 | 132 log.debug(f'{INDENT*7} new binding from {self} -> {outBinding}') |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
133 return True |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
134 return False |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
135 |
| 1639 | 136 def _advanceWithFunctions(self) -> bool: |
| 1637 | 137 pred: Node = self.lhsStmt[1] |
| 1640 | 138 if not isinstance(pred, URIRef): |
| 139 raise NotImplementedError | |
| 1632 | 140 |
| 1637 | 141 for functionType in functionsFor(pred): |
| 142 fn = functionType(self.lhsStmt, self.parent.graph) | |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
143 try: |
| 1637 | 144 out = fn.bind(self._prevBindings()) |
| 145 except BindingUnknown: | |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
146 pass |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
147 else: |
| 1637 | 148 if out is not None: |
| 149 binding: CandidateBinding = self._prevBindings().copy() | |
| 150 binding.addNewBindings(out) | |
| 151 if binding not in self._seenBindings: | |
| 152 self._seenBindings.append(binding) | |
| 153 self._current = binding | |
| 154 log.debug(f'{INDENT*7} new binding from {self} -> {binding}') | |
| 155 return True | |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
156 |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
157 return False |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
158 |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
159 def _boundOperands(self, operands) -> List[Node]: |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
160 pb: CandidateBinding = self._prevBindings() |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
161 |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
162 boundOperands: List[Node] = [] |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
163 for op in operands: |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
164 if isinstance(op, (Variable, BNode)): |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
165 boundOperands.append(pb.applyTerm(op)) |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
166 else: |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
167 boundOperands.append(op) |
|
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
168 return boundOperands |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
169 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
170 def _totalBindingIfThisStmtWereTrue(self, newStmt: Triple) -> CandidateBinding: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
171 outBinding = self._prevBindings().copy() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
172 for rt, ct in zip(self.lhsStmt, newStmt): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
173 if isinstance(rt, (Variable, BNode)): |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
174 if outBinding.contains(rt) and outBinding.applyTerm(rt) != ct: |
| 1648 | 175 msg = f'{rt=} {ct=} {outBinding=}' if log.isEnabledFor(logging.DEBUG) else '' |
| 176 raise Inconsistent(msg) | |
|
1635
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
177 outBinding.addNewBindings(CandidateBinding({rt: ct})) |
|
22d481f0a924
refactor: use CandidateBinding throughout, not loose dicts
drewp@bigasterisk.com
parents:
1634
diff
changeset
|
178 return outBinding |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
179 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
180 def currentBinding(self) -> CandidateBinding: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
181 if self.pastEnd(): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
182 raise NotImplementedError() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
183 return self._current |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
184 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
185 def pastEnd(self) -> bool: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
186 return self._pastEnd |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
187 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
188 def restart(self): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
189 self._pastEnd = False |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
190 self._seenBindings = [] |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
191 self.advance() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
192 if self.pastEnd(): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
193 raise NoOptions() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
194 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
195 |
| 1594 | 196 @dataclass |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
197 class Lhs: |
| 1648 | 198 graph: Graph # our full LHS graph, as input. See below for the statements partitioned into groups. |
| 1594 | 199 |
| 200 def __post_init__(self): | |
| 1648 | 201 |
| 202 usedByFuncs: Set[Triple] = lhsStmtsUsedByFuncs(self.graph) | |
| 203 | |
| 204 stmtsToMatch = list(self.graph - usedByFuncs) | |
| 205 self.staticStmts = [] | |
| 206 self.patternStmts = [] | |
| 207 for st in stmtsToMatch: | |
| 208 if all(isinstance(term, (URIRef, Literal)) for term in st): | |
| 209 self.staticStmts.append(st) | |
| 210 else: | |
| 211 self.patternStmts.append(st) | |
| 212 | |
| 213 # sort them by variable dependencies; don't just try all perms! | |
| 214 def lightSortKey(stmt): # Not this. | |
| 215 (s, p, o) = stmt | |
| 216 return p in rulePredicates(), p, s, o | |
| 217 | |
| 218 self.patternStmts.sort(key=lightSortKey) | |
| 219 | |
| 220 self.myPreds = set(p for s, p, o in self.graph if isinstance(p, URIRef)) | |
| 221 self.myPreds -= rulePredicates() | |
| 222 self.myPreds -= {RDF.first, RDF.rest} | |
| 223 self.myPreds = set(self.myPreds) | |
| 1602 | 224 |
|
1609
34f2817320cc
new tests for a smaller part of the inner loop
drewp@bigasterisk.com
parents:
1608
diff
changeset
|
225 def __repr__(self): |
|
34f2817320cc
new tests for a smaller part of the inner loop
drewp@bigasterisk.com
parents:
1608
diff
changeset
|
226 return f"Lhs({graphDump(self.graph)})" |
|
34f2817320cc
new tests for a smaller part of the inner loop
drewp@bigasterisk.com
parents:
1608
diff
changeset
|
227 |
| 1648 | 228 def findCandidateBindings(self, knownTrue: ReadOnlyWorkingSet, stats, ruleStatementsIterationLimit) -> Iterator['BoundLhs']: |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
229 """bindings that fit the LHS of a rule, using statements from workingSet and functions |
|
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
230 from LHS""" |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
231 if self.graph.__len__() == 0: |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
232 # special case- no LHS! |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
233 yield BoundLhs(self, CandidateBinding({})) |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
234 return |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
235 |
| 1640 | 236 if self._checkPredicateCounts(knownTrue): |
| 237 stats['_checkPredicateCountsCulls'] += 1 | |
| 238 return | |
| 239 | |
| 1648 | 240 if not all(st in knownTrue for st in self.staticStmts): |
| 241 stats['staticStmtCulls'] += 1 | |
| 242 return | |
| 243 | |
| 244 if len(self.patternStmts) == 0: | |
| 245 # static only | |
| 246 yield BoundLhs(self, CandidateBinding({})) | |
| 247 return | |
| 248 | |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
249 log.debug(f'{INDENT*4} build new StmtLooper stack') |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
250 |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
251 try: |
| 1648 | 252 stmtStack = self._assembleRings(knownTrue, stats) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
253 except NoOptions: |
| 1632 | 254 log.debug(f'{INDENT*5} start up with no options; 0 bindings') |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
255 return |
| 1632 | 256 self._debugStmtStack('initial odometer', stmtStack) |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
257 self._assertAllRingsAreValid(stmtStack) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
258 |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
259 lastRing = stmtStack[-1] |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
260 iterCount = 0 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
261 while True: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
262 iterCount += 1 |
| 1648 | 263 if iterCount > ruleStatementsIterationLimit: |
| 264 raise ValueError('rule too complex') | |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
265 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
266 log.debug(f'{INDENT*4} vv findCandBindings iteration {iterCount}') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
267 |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
268 yield BoundLhs(self, lastRing.currentBinding()) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
269 |
| 1632 | 270 self._debugStmtStack('odometer', stmtStack) |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
271 |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
272 done = self._advanceAll(stmtStack) |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
273 |
| 1632 | 274 self._debugStmtStack('odometer after ({done=})', stmtStack) |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
275 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
276 log.debug(f'{INDENT*4} ^^ findCandBindings iteration done') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
277 if done: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
278 break |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
279 |
| 1632 | 280 def _debugStmtStack(self, label, stmtStack): |
| 281 log.debug(f'{INDENT*5} {label}:') | |
| 282 for l in stmtStack: | |
| 283 log.debug(f'{INDENT*6} {l} curbind={l.currentBinding() if not l.pastEnd() else "<end>"}') | |
| 284 | |
| 1640 | 285 def _checkPredicateCounts(self, knownTrue): |
| 286 """raise NoOptions quickly in some cases""" | |
| 1648 | 287 |
| 288 if any((None, p, None) not in knownTrue for p in self.myPreds): | |
| 1640 | 289 return True |
| 1648 | 290 log.info(f'{INDENT*2} checkPredicateCounts does not cull because all {self.myPreds=} are in knownTrue') |
| 1640 | 291 return False |
| 292 | |
| 1648 | 293 def _assembleRings(self, knownTrue: ReadOnlyWorkingSet, stats) -> List[StmtLooper]: |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
294 """make StmtLooper for each stmt in our LHS graph, but do it in a way that they all |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
295 start out valid (or else raise NoOptions)""" |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
296 |
| 1648 | 297 log.info(f'{INDENT*2} stats={dict(stats)}') |
| 298 log.info(f'{INDENT*2} taking permutations of {len(self.patternStmts)=}') | |
| 299 for i, perm in enumerate(itertools.permutations(self.patternStmts)): | |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
300 stmtStack: List[StmtLooper] = [] |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
301 prev: Optional[StmtLooper] = None |
| 1648 | 302 if log.isEnabledFor(logging.DEBUG): |
| 303 log.debug(f'{INDENT*5} [perm {i}] try stmts in this order: {" -> ".join(graphDump([p]) for p in perm)}') | |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
304 |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
305 for s in perm: |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
306 try: |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
307 elem = StmtLooper(s, prev, knownTrue, parent=self) |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
308 except NoOptions: |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
309 log.debug(f'{INDENT*6} permutation didnt work, try another') |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
310 break |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
311 stmtStack.append(elem) |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
312 prev = stmtStack[-1] |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
313 else: |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
314 return stmtStack |
| 1648 | 315 if i > 5000: |
| 316 raise NotImplementedError(f'trying too many permutations {len(self.patternStmts)=}') | |
| 317 | |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
318 log.debug(f'{INDENT*6} no perms worked- rule cannot match anything') |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
319 raise NoOptions() |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
320 |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
321 def _advanceAll(self, stmtStack: List[StmtLooper]) -> bool: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
322 carry = True # 1st elem always must advance |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
323 for i, ring in enumerate(stmtStack): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
324 # unlike normal odometer, advancing any earlier ring could invalidate later ones |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
325 if carry: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
326 log.debug(f'{INDENT*5} advanceAll [{i}] {ring} carry/advance') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
327 ring.advance() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
328 carry = False |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
329 if ring.pastEnd(): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
330 if ring is stmtStack[-1]: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
331 log.debug(f'{INDENT*5} advanceAll [{i}] {ring} says we done') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
332 return True |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
333 log.debug(f'{INDENT*5} advanceAll [{i}] {ring} restart') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
334 ring.restart() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
335 carry = True |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
336 return False |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
337 |
|
1633
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
338 def _assertAllRingsAreValid(self, stmtStack): |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
339 if any(ring.pastEnd() for ring in stmtStack): # this is an unexpected debug assertion |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
340 log.debug(f'{INDENT*5} some rings started at pastEnd {stmtStack}') |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
341 raise NoOptions() |
|
6107603ed455
fix farenheit rule case, fix some others that depend on rings order, but this breaks some performance because of itertools.perm
drewp@bigasterisk.com
parents:
1632
diff
changeset
|
342 |
|
1592
d7b66234064b
pure reordering of funcs to make the next diffs smaller
drewp@bigasterisk.com
parents:
1591
diff
changeset
|
343 |
|
1622
38bd8ef9ef67
add CandidateTermMatches, unused so far
drewp@bigasterisk.com
parents:
1621
diff
changeset
|
344 @dataclass |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
345 class BoundLhs: |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
346 lhs: Lhs |
| 1610 | 347 binding: CandidateBinding |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
348 |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
349 |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
350 @dataclass |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
351 class Rule: |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
352 lhsGraph: Graph |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
353 rhsGraph: Graph |
| 1632 | 354 |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
355 def __post_init__(self): |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
356 self.lhs = Lhs(self.lhsGraph) |
| 1632 | 357 # |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
358 self.rhsBnodeMap = {} |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
359 |
| 1648 | 360 def applyRule(self, workingSet: Graph, implied: Graph, stats: Dict, ruleStatementsIterationLimit): |
| 361 for bound in self.lhs.findCandidateBindings(ReadOnlyGraphAggregate([workingSet]), stats, ruleStatementsIterationLimit): | |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
362 log.debug(f'{INDENT*5} +rule has a working binding: {bound}') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
363 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
364 # rhs could have more bnodes, and they just need to be distinct per rule-firing that we do |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
365 existingRhsBnodes = set() |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
366 for stmt in self.rhsGraph: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
367 for t in stmt: |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
368 if isinstance(t, BNode): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
369 existingRhsBnodes.add(t) |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
370 # if existingRhsBnodes: |
| 1632 | 371 # log.debug(f'{INDENT*6} mapping rhs bnodes {existingRhsBnodes} to new ones') |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
372 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
373 for b in existingRhsBnodes: |
|
1612
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
374 |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
375 key = tuple(sorted(bound.binding.binding.items())), b |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
376 self.rhsBnodeMap.setdefault(key, BNode()) |
|
1638
0ba1625037ae
don't crash, just skip the rule if there's a BindingConflict (no clear test case yet)
drewp@bigasterisk.com
parents:
1637
diff
changeset
|
377 try: |
|
0ba1625037ae
don't crash, just skip the rule if there's a BindingConflict (no clear test case yet)
drewp@bigasterisk.com
parents:
1637
diff
changeset
|
378 bound.binding.addNewBindings(CandidateBinding({b: self.rhsBnodeMap[key]})) |
|
0ba1625037ae
don't crash, just skip the rule if there's a BindingConflict (no clear test case yet)
drewp@bigasterisk.com
parents:
1637
diff
changeset
|
379 except BindingConflict: |
|
0ba1625037ae
don't crash, just skip the rule if there's a BindingConflict (no clear test case yet)
drewp@bigasterisk.com
parents:
1637
diff
changeset
|
380 continue |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
381 |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
382 # for lhsBoundStmt in bound.binding.apply(bound.lhsStmtsWithoutEvals()): |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
383 # log.debug(f'{INDENT*6} adding to workingSet {lhsBoundStmt=}') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
384 # workingSet.add(lhsBoundStmt) |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
385 # log.debug(f'{INDENT*6} rhsGraph is good: {list(self.rhsGraph)}') |
|
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
386 |
|
1612
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
387 for newStmt in bound.binding.apply(self.rhsGraph): |
|
1631
2c85a4f5dd9c
big rewrite of infer() using statements not variables as the things to iterate over
drewp@bigasterisk.com
parents:
1627
diff
changeset
|
388 # log.debug(f'{INDENT*6} adding {newStmt=}') |
|
1612
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
389 workingSet.add(newStmt) |
|
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
390 implied.add(newStmt) |
|
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
391 |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
392 |
| 1648 | 393 @dataclass |
| 1587 | 394 class Inference: |
| 1648 | 395 rulesIterationLimit = 3 |
| 396 ruleStatementsIterationLimit = 3 | |
| 1587 | 397 |
| 398 def __init__(self) -> None: | |
| 1648 | 399 self.rules: List[Rule] = [] |
| 400 self._nonRuleStmts: List[Triple] = [] | |
| 1587 | 401 |
| 402 def setRules(self, g: ConjunctiveGraph): | |
| 1648 | 403 self.rules = [] |
| 404 self._nonRuleStmts = [] | |
|
1599
abbf0eb0e640
fix a bug with a slightly moer complicated set of rules
drewp@bigasterisk.com
parents:
1598
diff
changeset
|
405 for stmt in g: |
|
abbf0eb0e640
fix a bug with a slightly moer complicated set of rules
drewp@bigasterisk.com
parents:
1598
diff
changeset
|
406 if stmt[1] == LOG['implies']: |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
407 self.rules.append(Rule(stmt[0], stmt[2])) |
|
1612
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
408 # other stmts should go to a default working set? |
| 1587 | 409 |
| 1601 | 410 @INFER_CALLS.time() |
| 1587 | 411 def infer(self, graph: Graph): |
| 412 """ | |
| 413 returns new graph of inferred statements. | |
| 414 """ | |
| 1626 | 415 n = graph.__len__() |
| 416 INFER_GRAPH_SIZE.observe(n) | |
| 417 log.info(f'{INDENT*0} Begin inference of graph len={n} with rules len={len(self.rules)}:') | |
| 1601 | 418 startTime = time.time() |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
419 stats: Dict[str, Union[int, float]] = defaultdict(lambda: 0) |
|
1589
5c1055be3c36
WIP more debugging, working towards bnode-matching support
drewp@bigasterisk.com
parents:
1588
diff
changeset
|
420 # everything that is true: the input graph, plus every rule conclusion we can make |
|
1593
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
421 workingSet = Graph() |
|
b0df43d5494c
big rewrite- more classes, smaller methods, more typesafe, all current tests passing
drewp@bigasterisk.com
parents:
1592
diff
changeset
|
422 workingSet += graph |
| 1587 | 423 |
| 1594 | 424 # just the statements that came from RHS's of rules that fired. |
| 1587 | 425 implied = ConjunctiveGraph() |
| 426 | |
| 1648 | 427 rulesIterations = 0 |
| 1587 | 428 delta = 1 |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
429 stats['initWorkingSet'] = cast(int, workingSet.__len__()) |
| 1648 | 430 while delta > 0 and rulesIterations <= self.rulesIterationLimit: |
| 1620 | 431 log.debug('') |
| 1648 | 432 log.info(f'{INDENT*1}*iteration {rulesIterations}') |
| 433 | |
| 1587 | 434 delta = -len(implied) |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
435 self._iterateAllRules(workingSet, implied, stats) |
| 1587 | 436 delta += len(implied) |
| 1648 | 437 rulesIterations += 1 |
| 1597 | 438 log.info(f'{INDENT*2} this inference iteration added {delta} more implied stmts') |
| 1648 | 439 stats['iterations'] = rulesIterations |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
440 stats['timeSpent'] = round(time.time() - startTime, 3) |
|
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
441 stats['impliedStmts'] = len(implied) |
| 1648 | 442 log.info(f'{INDENT*0} Inference done {dict(stats)}.') |
| 443 log.debug('Implied:') | |
| 444 log.debug(graphDump(implied)) | |
| 1587 | 445 return implied |
| 446 | |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
447 def _iterateAllRules(self, workingSet: Graph, implied: Graph, stats): |
|
1612
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
448 for i, rule in enumerate(self.rules): |
|
272f78d4671a
mark skipped tests. move applyRule into Rule. minor cleanups.
drewp@bigasterisk.com
parents:
1611
diff
changeset
|
449 self._logRuleApplicationHeader(workingSet, i, rule) |
| 1648 | 450 rule.applyRule(workingSet, implied, stats, self.ruleStatementsIterationLimit) |
| 1587 | 451 |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
452 def _logRuleApplicationHeader(self, workingSet, i, r: Rule): |
| 1594 | 453 if not log.isEnabledFor(logging.DEBUG): |
| 454 return | |
| 455 | |
| 456 log.debug('') | |
| 457 log.debug(f'{INDENT*2} workingSet:') | |
| 1648 | 458 # for j, stmt in enumerate(sorted(workingSet)): |
| 459 # log.debug(f'{INDENT*3} ({j}) {stmt}') | |
| 460 log.debug(f'{INDENT*3} {graphDump(workingSet, oneLine=False)}') | |
| 1594 | 461 |
| 462 log.debug('') | |
| 463 log.debug(f'{INDENT*2}-applying rule {i}') | |
| 1632 | 464 log.debug(f'{INDENT*3} rule def lhs:') |
|
1634
ba59cfc3c747
hack math:sum in there. Test suite is passing except some slow performers
drewp@bigasterisk.com
parents:
1633
diff
changeset
|
465 for stmt in sorted(r.lhsGraph, reverse=True): |
| 1632 | 466 log.debug(f'{INDENT*4} {stmt}') |
|
1607
b21885181e35
more modules, types. Maybe less repeated computation on BoundLhs
drewp@bigasterisk.com
parents:
1605
diff
changeset
|
467 log.debug(f'{INDENT*3} rule def rhs: {graphDump(r.rhsGraph)}') |
| 1594 | 468 |
|
1588
0757fafbfdab
WIP inferencer - partial var and function support
drewp@bigasterisk.com
parents:
1587
diff
changeset
|
469 |
| 1648 | 470 def graphDump(g: Union[Graph, List[Triple]], oneLine=True): |
| 1637 | 471 # this is very slow- debug only! |
| 472 if not log.isEnabledFor(logging.DEBUG): | |
| 473 return "(skipped dump)" | |
|
1590
327202020892
WIP inference- getting into more degenerate test cases
drewp@bigasterisk.com
parents:
1589
diff
changeset
|
474 if not isinstance(g, Graph): |
|
327202020892
WIP inference- getting into more degenerate test cases
drewp@bigasterisk.com
parents:
1589
diff
changeset
|
475 g2 = Graph() |
| 1594 | 476 g2 += g |
|
1590
327202020892
WIP inference- getting into more degenerate test cases
drewp@bigasterisk.com
parents:
1589
diff
changeset
|
477 g = g2 |
|
1589
5c1055be3c36
WIP more debugging, working towards bnode-matching support
drewp@bigasterisk.com
parents:
1588
diff
changeset
|
478 g.bind('', ROOM) |
|
5c1055be3c36
WIP more debugging, working towards bnode-matching support
drewp@bigasterisk.com
parents:
1588
diff
changeset
|
479 g.bind('ex', Namespace('http://example.com/')) |
|
5c1055be3c36
WIP more debugging, working towards bnode-matching support
drewp@bigasterisk.com
parents:
1588
diff
changeset
|
480 lines = cast(bytes, g.serialize(format='n3')).decode('utf8').splitlines() |
| 1648 | 481 lines = [line for line in lines if not line.startswith('@prefix')] |
| 482 if oneLine: | |
| 483 lines = [line.strip() for line in lines] | |
|
1589
5c1055be3c36
WIP more debugging, working towards bnode-matching support
drewp@bigasterisk.com
parents:
1588
diff
changeset
|
484 return ' '.join(lines) |