Mercurial > code > home > repos > streamed-graph

changeset 8:6fefd287aff9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
closer- element now holds a changing graph, but can't draw it yet
author drewp@bigasterisk.com
date Thu, 05 Dec 2019 01:32:13 -0800
parents daf08d794660
children 26d3e4860adc
files index.html src/graph_view.js src/graph_view.ts src/json_ld_quads.ts src/streamed-graph.ts src/streamed_graph_client.ts webpack-dev.config.ts
diffstat 7 files changed, 262 insertions(+), 278 deletions(-) [+]
line wrap: on
line diff
--- a/index.html	Wed Dec 04 23:44:29 2019 -0800
+++ b/index.html	Thu Dec 05 01:32:13 2019 -0800
@@ -4,6 +4,6 @@
         <h1>streamed-graph demo</h1>
 
         <script src="./build/streamed-graph.bundle.js"></script>
-        <streamed-graph url="something"></streamed-graph>
+        <streamed-graph url="http://bang5:9075/graph/events"></streamed-graph>
     </body>
 </html>
\ No newline at end of file
--- a/src/graph_view.js	Wed Dec 04 23:44:29 2019 -0800
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,202 +0,0 @@
-// from /my/site/homepage/www/rdf/browse/graphView.js
-
-
-import { SuffixLabels } from './suffixLabels.js';
-
-const groupByRdfType = (graph) => {
-    const env = graph.store.rdf;
-    const rdfType = env.createNamedNode('rdf:type');
-    const byType = new Map(); // type : subjs
-    const untyped = new Set(); // subjs
-    graph.quadStore.quads({}, (q) => {
-        let subjType = null;
-        graph.quadStore.quads({subject: q.subject,
-                               predicate: rdfType},
-                              (q2) => { subjType = q2.object; });
-        if (subjType){
-            subjType = subjType.toString();
-            if (!byType.has(subjType)) {
-                byType.set(subjType, new Set());
-            }
-            byType.get(subjType).add(q.subject.toString());
-        } else {
-            untyped.add(q.subject.toString());
-        }
-
-    });
-    return {byType: byType, untyped: untyped};
-};
-
-const graphView = (graph) => {
-    const env = graph.store.rdf;
-
-    const labels = new SuffixLabels();
-    graph.quadStore.quads({}, (q) => {
-        if (q.subject.interfaceName == "NamedNode") { labels.planDisplayForNode(q.subject); }
-        if (q.predicate.interfaceName == "NamedNode") { labels.planDisplayForNode(q.predicate); }
-        if (q.object.interfaceName == "NamedNode") { labels.planDisplayForNode(q.object); }
-        if (q.object.interfaceName == "Literal" && q.object.datatype) { labels.planDisplayForNode(env.createNamedNode(q.object.datatype)); }
-    });
-
-    const rdfNode = (n) => {
-        if (n.interfaceName == "Literal") {
-            let dtPart = "";
-            if (n.datatype) {
-                dtPart = html`
-        ^^<span class="literalType">
-          ${rdfNode(env.createNamedNode(n.datatype))}
-        </span>`;
-            }
-            return html`<span class="literal">${n.nominalValue}${dtPart}</span>`;
-        }
-        if (n.interfaceName == "NamedNode") {
-            let dn = labels.getLabelForNode(n);
-            if (dn.match(/XMLSchema#.*/)) { dn = dn.replace('XMLSchema#', 'xsd:'); }
-            if (dn.match(/rdf-schema#.*/)) { dn = dn.replace('rdf-schema#', 'rdfs:'); }
-            return html`<a class="graphUri" href="${n.toString()}">${dn}</a>`;
-        }
-
-        return html`[${n.interfaceName} ${n.toNT()}]`;
-    }
-
-    const objBlock = (obj) => {
-        return html`
-        <div class="object">
-          ${rdfNode(obj)} <!-- indicate what source or graph said this stmt -->
-        </div>
-    `;
-    };
-
-    /// bunch of table rows
-    const predBlock = (subj, pred) => {
-        const objsSet = new Set();
-        graph.quadStore.quads({ subject: subj, predicate: pred }, (q) => {
-
-            if (q.object.length) {
-                console.log(q.object)
-            }
-            objsSet.add(q.object);
-        });
-        const objs = Array.from(objsSet.values());
-        objs.sort();
-        return html`
-      <div class="predicate">${rdfNode(pred)}
-        <div>
-          ${objs.map(objBlock)}
-        </div>
-      </div>
-    `;
-    };
-
-    const {byType, untyped} = groupByRdfType(graph);
-    const typedSubjs = Array.from(byType.keys());
-    typedSubjs.sort();
-
-    const untypedSubjs = Array.from(untyped.values());
-    untypedSubjs.sort();
-
-    const subjBlock = (subj) => {
-        const subjNode = env.createNamedNode(subj);
-        const predsSet = new Set();
-        graph.quadStore.quads({ subject: subjNode }, (q) => {
-            predsSet.add(q.predicate);
-        });
-        const preds = Array.from(predsSet.values());
-        preds.sort();
-        return html`
-      <div class="subject">${rdfNode(subjNode)}
-        <!-- todo: special section for uri/type-and-icon/label/comment -->
-        <div>
-          ${preds.map((p) => { return predBlock(subjNode, p); })}
-        </div>
-      </div>
-    `;
-    };
-    const byTypeBlock = (typeUri) => {
-        const subjs = Array.from(byType.get(typeUri));
-        subjs.sort();
-
-        const graphCells = new Map(); // [subj, pred] : objs
-        const preds = new Set();
-
-        subjs.forEach((subj) => {
-            graph.quadStore.quads({subject: env.createNamedNode(subj)}, (q) => {
-                preds.add(q.predicate.toString());
-                const cellKey = subj + '|||' + q.predicate.toString();
-                if (!graphCells.has(cellKey)) {
-                    graphCells.set(cellKey, new Set());
-                }
-                graphCells.get(cellKey).add(q.object);
-            });
-        });
-        const predsList = Array.from(preds);
-        predsList.splice(predsList.indexOf('http://www.w3.org/1999/02/22-rdf-syntax-ns#type'), 1);
-        // also pull out label, which should be used on 1st column
-        predsList.sort();
-
-        const thead = () => {
-            const predColumnHead = (pred) => {
-                return html`<th>${rdfNode(env.createNamedNode(pred))}</th>`;
-            };
-            return html`
-              <thead>
-                <tr>
-                  <th></th>
-                  ${predsList.map(predColumnHead)}
-                </tr>
-              </thead>`;
-        };
-
-        const instanceRow = (subj) => {
-            const cell = (pred) => {
-                const objs = graphCells.get(subj + '|||' + pred);
-                if (!objs) {
-                    return html`<td></td>`;
-                }
-                const objsList = Array.from(objs);
-                objsList.sort();
-                const draw = (obj) => {
-                    return html`<div>${rdfNode(obj)}</div>`
-                };
-                return html`<td>${objsList.map(draw)}</td>`;
-            };
-
-            return html`
-              <tr>
-                <td>${rdfNode(env.createNamedNode(subj))}</td>
-                ${predsList.map(cell)}
-              </tr>
-            `;
-        };
-
-        return html`
-          <div>[icon] ${rdfNode(env.createNamedNode(typeUri))} resources</div>
-<div class="typeBlockScroll">
-          <table class="typeBlock">
-            ${thead()}
-            ${subjs.map(instanceRow)}
-          </table>
-</div>
-        `;
-    };
-
-    return html`
-      <link rel="stylesheet" href="/rdf/browse/style.css">
-
-      <section>
-        <h2>
-          Current graph (<a href="${graph.events.url}">${graph.events.url}</a>)
-        </h2>
-        <div>
-         <!-- todo: graphs and provenance.
-          These statements are all in the
-          <span data-bind="html: $root.createCurie(graphUri())">...</span> graph.-->
-        </div>
-        ${typedSubjs.map(byTypeBlock)}
-        <div class="spoGrid">
-          ${untypedSubjs.map(subjBlock)}
-        </div>
-      </section>
-    `;
-}
-export { graphView }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/graph_view.ts	Thu Dec 05 01:32:13 2019 -0800
@@ -0,0 +1,206 @@
+// from /my/site/homepage/www/rdf/browse/graphView.js
+
+import { html } from 'lit-html';
+import { SuffixLabels } from './suffixLabels.js';
+import { Store, Quad, DataFactory } from "n3"
+const { namedNode, literal, quad } = DataFactory;
+
+const groupByRdfType = (graph: Store) => {
+  const env = graph.store.rdf;
+  const rdfType = env.createNamedNode('rdf:type');
+  const byType = new Map(); // type : subjs
+  const untyped = new Set(); // subjs
+  graph.quadStore.quads({}, (q) => {
+    let subjType = null;
+    graph.quadStore.quads({
+      subject: q.subject,
+      predicate: rdfType
+    },
+      (q2) => { subjType = q2.object; });
+    if (subjType) {
+      subjType = subjType.toString();
+      if (!byType.has(subjType)) {
+        byType.set(subjType, new Set());
+      }
+      byType.get(subjType).add(q.subject.toString());
+    } else {
+      untyped.add(q.subject.toString());
+    }
+
+  });
+  return { byType: byType, untyped: untyped };
+};
+
+const graphView = (graph: Store) => {
+  const env = graph.store.rdf;
+
+  const labels = new SuffixLabels();
+  graph.quadStore.quads({}, (q) => {
+    if (q.subject.interfaceName == "NamedNode") { labels.planDisplayForNode(q.subject); }
+    if (q.predicate.interfaceName == "NamedNode") { labels.planDisplayForNode(q.predicate); }
+    if (q.object.interfaceName == "NamedNode") { labels.planDisplayForNode(q.object); }
+    if (q.object.interfaceName == "Literal" && q.object.datatype) { labels.planDisplayForNode(env.createNamedNode(q.object.datatype)); }
+  });
+
+  const rdfNode = (n) => {
+    if (n.interfaceName == "Literal") {
+      let dtPart: any = "";
+      if (n.datatype) {
+        dtPart = html`
+        ^^<span class="literalType">
+          ${rdfNode(env.createNamedNode(n.datatype))}
+        </span>`;
+      }
+      return html`<span class="literal">${n.nominalValue}${dtPart}</span>`;
+    }
+    if (n.interfaceName == "NamedNode") {
+      let dn = labels.getLabelForNode(n);
+      if (dn.match(/XMLSchema#.*/)) { dn = dn.replace('XMLSchema#', 'xsd:'); }
+      if (dn.match(/rdf-schema#.*/)) { dn = dn.replace('rdf-schema#', 'rdfs:'); }
+      return html`<a class="graphUri" href="${n.toString()}">${dn}</a>`;
+    }
+
+    return html`[${n.interfaceName} ${n.toNT()}]`;
+  }
+
+  const objBlock = (obj) => {
+    return html`
+        <div class="object">
+          ${rdfNode(obj)} <!-- indicate what source or graph said this stmt -->
+        </div>
+    `;
+  };
+
+  /// bunch of table rows
+  const predBlock = (subj, pred) => {
+    const objsSet = new Set();
+    graph.quadStore.quads({ subject: subj, predicate: pred }, (q) => {
+
+      if (q.object.length) {
+        console.log(q.object)
+      }
+      objsSet.add(q.object);
+    });
+    const objs = Array.from(objsSet.values());
+    objs.sort();
+    return html`
+      <div class="predicate">${rdfNode(pred)}
+        <div>
+          ${objs.map(objBlock)}
+        </div>
+      </div>
+    `;
+  };
+
+  const { byType, untyped } = groupByRdfType(graph);
+  const typedSubjs = Array.from(byType.keys());
+  typedSubjs.sort();
+
+  const untypedSubjs = Array.from(untyped.values());
+  untypedSubjs.sort();
+
+  const subjBlock = (subj) => {
+    const subjNode = env.createNamedNode(subj);
+    const predsSet = new Set();
+    graph.quadStore.quads({ subject: subjNode }, (q) => {
+      predsSet.add(q.predicate);
+    });
+    const preds = Array.from(predsSet.values());
+    preds.sort();
+    return html`
+      <div class="subject">${rdfNode(subjNode)}
+        <!-- todo: special section for uri/type-and-icon/label/comment -->
+        <div>
+          ${preds.map((p) => { return predBlock(subjNode, p); })}
+        </div>
+      </div>
+    `;
+  };
+  const byTypeBlock = (typeUri) => {
+    const subjs = Array.from(byType.get(typeUri));
+    subjs.sort();
+
+    const graphCells = new Map(); // [subj, pred] : objs
+    const preds = new Set();
+
+    subjs.forEach((subj) => {
+      graph.quadStore.quads({ subject: env.createNamedNode(subj) }, (q) => {
+        preds.add(q.predicate.toString());
+        const cellKey = subj + '|||' + q.predicate.toString();
+        if (!graphCells.has(cellKey)) {
+          graphCells.set(cellKey, new Set());
+        }
+        graphCells.get(cellKey).add(q.object);
+      });
+    });
+    const predsList = Array.from(preds);
+    predsList.splice(predsList.indexOf('http://www.w3.org/1999/02/22-rdf-syntax-ns#type'), 1);
+    // also pull out label, which should be used on 1st column
+    predsList.sort();
+
+    const thead = () => {
+      const predColumnHead = (pred) => {
+        return html`<th>${rdfNode(env.createNamedNode(pred))}</th>`;
+      };
+      return html`
+              <thead>
+                <tr>
+                  <th></th>
+                  ${predsList.map(predColumnHead)}
+                </tr>
+              </thead>`;
+    };
+
+    const instanceRow = (subj) => {
+      const cell = (pred) => {
+        const objs = graphCells.get(subj + '|||' + pred);
+        if (!objs) {
+          return html`<td></td>`;
+        }
+        const objsList = Array.from(objs);
+        objsList.sort();
+        const draw = (obj) => {
+          return html`<div>${rdfNode(obj)}</div>`
+        };
+        return html`<td>${objsList.map(draw)}</td>`;
+      };
+
+      return html`
+              <tr>
+                <td>${rdfNode(env.createNamedNode(subj))}</td>
+                ${predsList.map(cell)}
+              </tr>
+            `;
+    };
+
+    return html`
+          <div>[icon] ${rdfNode(env.createNamedNode(typeUri))} resources</div>
+<div class="typeBlockScroll">
+          <table class="typeBlock">
+            ${thead()}
+            ${subjs.map(instanceRow)}
+          </table>
+</div>
+        `;
+  };
+
+  return html`
+      <link rel="stylesheet" href="/rdf/browse/style.css">
+
+      <section>
+        <h2>
+          Current graph (<a href="${graph.events.url}">${graph.events.url}</a>)
+        </h2>
+        <div>
+         <!-- todo: graphs and provenance.
+          These statements are all in the
+          <span data-bind="html: $root.createCurie(graphUri())">...</span> graph.-->
+        </div>
+        ${typedSubjs.map(byTypeBlock)}
+        <div class="spoGrid">
+          ${untypedSubjs.map(subjBlock)}
+        </div>
+      </section>
+    `;
+}
+export { graphView }
--- a/src/json_ld_quads.ts	Wed Dec 04 23:44:29 2019 -0800
+++ b/src/json_ld_quads.ts	Thu Dec 05 01:32:13 2019 -0800
@@ -1,46 +1,34 @@
-import { Quad, Node } from "./rdf_types";
 import * as jsonld from "jsonld";
+import { DataFactory, Quad } from 'n3';
+const { namedNode, literal, quad } = DataFactory;
+import ns from 'n3/src/IRIs';
+const {rdf} = ns;
 
-function quadFromExpandedStatement(rdfEnv: any, subj: any, pred: string, obj: any, graphNode: any): Quad {
-    return {
-        subject: rdfEnv.createNamedNode(subj['@id']),
-        predicate: rdfEnv.createNamedNode(pred),
-        object: (obj['@id'] ? rdfEnv.createNamedNode(obj['@id']) :
-            rdfEnv.createLiteral(obj['@value'], obj['@language'], obj['@type'])),
-        graph: graphNode,
-    };
-}
-function quadFromTypeStatement(rdfEnv: any, subj: any, obj: any, graphNode: any): Quad {
-    return {
-        subject: rdfEnv.createNamedNode(subj['@id']),
-        predicate: rdfEnv.createNamedNode('rdf:type'),
-        object: rdfEnv.createNamedNode(obj),
-        graph: graphNode,
-    };
-}
-
-export function eachJsonLdQuad(rdfEnv: any, jsonLdObj: object, onQuad: (Quad) => void, done: () => void) {
+export function eachJsonLdQuad(jsonLdObj: object, onQuad: (q: Quad) => void, done: () => void) {
     jsonld.expand(jsonLdObj, function onExpand(err, expanded) {
         if (err) {
             throw new Error();
         }
         (expanded as [object]).forEach(function (g) {
             var graph = g['@id'];
-            var graphNode = rdfEnv.createNamedNode(graph) as Node;
+            var graphNode = namedNode(graph);
             g['@graph'].forEach(function (subj) {
+                const subjNode = namedNode(subj['@id']);
                 for (let pred in subj) {
-                    if (pred.match(/^[^@]/)) {
-                        subj[pred].forEach(function (obj) {
-                            onQuad(quadFromExpandedStatement(rdfEnv, subj, pred, obj, graphNode));
-                        });
+                    if (pred === '@id') {
+                        continue;
                     }
-                    else {
-                        if (pred === "@type") {
-                            subj[pred].forEach((obj) => {
-                                onQuad(quadFromTypeStatement(rdfEnv, subj, obj, graphNode));
-                            });
-                        }
+                    let predNode;
+                    if (pred === "@type") {
+                        predNode = namedNode(rdf.type);
+                    } else {
+                        predNode = namedNode(pred['@id']);
                     }
+                    subj[pred].forEach(function (obj) {
+                        const objNode = (obj['@id'] ? namedNode(obj['@id']) :
+                            literal(obj['@value'], obj['@language'] || obj['@type']));
+                        onQuad(quad(subjNode, predNode, objNode, graphNode));
+                    });
                 }
             });
         });
--- a/src/streamed-graph.ts	Wed Dec 04 23:44:29 2019 -0800
+++ b/src/streamed-graph.ts	Thu Dec 05 01:32:13 2019 -0800
@@ -6,21 +6,18 @@
 import { PolymerElement, html } from '@polymer/polymer';
 import { customElement, property, computed } from '@polymer/decorators';
 import { render } from 'lit-html';
-// import { graphView } from '/rdf/browse/graphView.js';
-
-
+import { graphView } from './graph_view';
+import { Store, DataFactory } from "n3"
 
 import { StreamedGraphClient } from './streamed_graph_client';
 
-console.log(StreamedGraphClient);
-
 @customElement('streamed-graph')
 class StreamedGraph extends PolymerElement {
     @property({ type: String })
     url: string = '';
 
     @property({ type: Object })
-    graph: Object;
+    graph: {version: number, graph: Store};
 
     @property({ type: Boolean })
     expanded: Boolean = false;
@@ -45,13 +42,15 @@
                 StreamedGraph <a href="{{url}}">[source]</a>:
                 {{status}}
             </div>
-            <div id="graphView">graph here
-            </div>`;
+            <div id="graphView"></div>`;
     }
 
     ready() {
         super.ready();
+        this.graph = {version: -1, graph: null};
         this.graphView = this.shadowRoot.getElementById("graphView");
+
+        this._onUrl(this.url); // todo: watch for changes and rebuild
     }
 
     toggleExpand(ev) {
@@ -70,17 +69,18 @@
     }
 
     _onUrl(url) {
-        // if (this.sg) { this.sg.close(); }
-        // this.sg = new StreamedGraphClient(
-        //     url, 
-        //     this.onGraphChanged.bind(this), 
-        //     this.set.bind(this, 'status'), 
-        //     [],//window.NS,
-        //     []);
+        if (this.sg) { this.sg.close(); }
+        this.sg = new StreamedGraphClient(
+            url,
+            this.onGraphChanged.bind(this),
+            this.set.bind(this, 'status'),
+            [],//window.NS,
+            []
+        );
     }
 
     onGraphChanged() {
-        //this.graph = { version: this.graph.version + 1, graph: this.sg };
+        this.graph = { version: this.graph.version + 1, graph: this.sg.store };
         if (this.expanded) {
             this.redrawGraph();
         }
@@ -88,7 +88,7 @@
 
     _redrawLater() {
         if (!this.graphViewDirty) return;
-        //render(graphView(this.graph.graph), this.graphView);
+        render(graphView(this.graph.graph), this.graphView);
         this.graphViewDirty = false;
     }
 
--- a/src/streamed_graph_client.ts	Wed Dec 04 23:44:29 2019 -0800
+++ b/src/streamed_graph_client.ts	Thu Dec 05 01:32:13 2019 -0800
@@ -1,15 +1,18 @@
 // from /my/site/homepage/www/rdf/streamed-graph.js
 
 import * as async from "async";
-import * as jsonld from "jsonld";
+// import * as jsonld from "jsonld";
 
-//import eachJsonLdQuad from "./json_ld_quads";
+import { eachJsonLdQuad } from "./json_ld_quads";
 import { Store, DataFactory } from "n3"
 
-/// <reference types="eventsource" />
-const EventSource = window.EventSource;
+// /// <reference types="eventsource" />
+// const EventSource = window.EventSource;
 
 export class StreamedGraphClient {
+    // holds a n3 Store, which is synced to a server-side
+    // store that sends patches over SSE
+
     onStatus: (msg: string) => void;
     onGraphChanged: () => void;
     store: Store;
@@ -21,8 +24,6 @@
         prefixes: Array<Record<string, string>>,
         staticGraphUrls: Array<string>) {
         console.log('new StreamedGraph', eventsUrl);
-        // holds a n3 Store, which is synced to a server-side
-        // store that sends patches over SSE
         this.onStatus = onStatus;
         this.onGraphChanged = onGraphChanged;
         this.onStatus('startup...');
@@ -60,7 +61,6 @@
             throw new Error("zombie eventsource");
         }
 
-
         this.events = new EventSource(eventsUrl);
 
         this.events.addEventListener('error', (ev) => {
@@ -77,7 +77,7 @@
         this.events.addEventListener('fullGraph', (ev) => {
             this.onStatus('sync- full graph update');
             let onReplaced = () => {
-                this.onStatus('synced');
+                this.onStatus(`synced ${this.store.size}`);
                 this.onGraphChanged();
             };
             this.replaceFullGraph(ev.data, onReplaced);
@@ -86,7 +86,7 @@
         this.events.addEventListener('patch', (ev) => {
             this.onStatus('sync- updating');
             let onPatched = () => {
-                this.onStatus('synced');
+                this.onStatus(`synced ${this.store.size}`);
                 this.onGraphChanged();
             };
             this.patchGraph(ev.data, onPatched);
@@ -95,25 +95,23 @@
     }
 
     replaceFullGraph(jsonLdText: string, done: () => void) {
-        // this.quadStore.clear();
-        // eachJsonLdQuad(this.store.rdf, JSON.parse(jsonLdText),
-        //     this.quadStore.add.bind(this.quadStore), function () {
-        //         done();
-        //     });
-        // or this.store.insert([quad], quad.graph, function() {});
+        this.store = new Store({});
+        eachJsonLdQuad(JSON.parse(jsonLdText),
+            this.store.addQuad.bind(this.store),
+            done);
     }
 
     patchGraph(patchJson: string, done: () => void) {
         var patch = JSON.parse(patchJson).patch;
 
         async.series([
-            // (done) => {
-            //     eachJsonLdQuad(this.store.rdf, patch.deletes,
-            //         this.quadStore.remove.bind(this.quadStore), done);
-            // },
             (done) => {
-                // eachJsonLdQuad(this.store.rdf, patch.adds,
-                //     this.quadStore.add.bind(this.quadStore), done);
+                eachJsonLdQuad(patch.deletes,
+                    this.store.removeQuad.bind(this.store), done);
+            },
+            (done) => {
+                eachJsonLdQuad(patch.adds,
+                    this.store.addQuad.bind(this.store), done);
             },
             /* seriesDone */ (done) => {
                 done();
--- a/webpack-dev.config.ts	Wed Dec 04 23:44:29 2019 -0800
+++ b/webpack-dev.config.ts	Thu Dec 05 01:32:13 2019 -0800
@@ -29,12 +29,6 @@
         liveReload: true, // doesn't work
         overlay: true,
         watchContentBase: true,
-        // proxy: {
-        //     '/rdf': {
-        //         target: 'https://bigasterisk.com/',
-        //         //pathRewrite: {'^/api' : ''}  
-        //     }
-        // },
     }
 };