Mercurial > code > home > repos > streamed-graph
changeset 3:a7ba8627a7b6
still trying to make imports work. add other files too
| author | drewp@bigasterisk.com |
|---|---|
| date | Wed, 04 Dec 2019 00:09:15 -0800 |
| parents | 6cd3aaf431b6 |
| children | a668a774b162 |
| files | package-lock.json package.json src/graph_view.js src/json_ld_quads.ts src/rdf_types.ts src/streamed-graph.ts src/streamed_graph_client.ts src/suffixLabels.js src/suffixLabels_test.html tsconfig.json webpack-dev.config.ts |
| diffstat | 11 files changed, 692 insertions(+), 1 deletions(-) [+] |
line wrap: on
line diff
--- a/package-lock.json Tue Dec 03 21:48:36 2019 -0800 +++ b/package-lock.json Wed Dec 04 00:09:15 2019 -0800 @@ -3908,6 +3908,11 @@ "thenify-all": "^1.0.0" } }, + "n3": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/n3/-/n3-1.3.4.tgz", + "integrity": "sha512-Eu5EVYGncuwiTlOV1J6p3OFBNSfI84D+fW0o8o5s2aRowO3yRcM4SvqPTOKzCCJutRvaXP0J9GIzwrP6tINm2Q==" + }, "nan": { "version": "2.14.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
--- a/package.json Tue Dec 03 21:48:36 2019 -0800 +++ b/package.json Wed Dec 04 00:09:15 2019 -0800 @@ -9,7 +9,8 @@ "@webcomponents/webcomponentsjs": "^2.4.0", "async": "^3.1.0", "jsonld": "^1.8.1", - "lit-html": "^1.1.2" + "lit-html": "^1.1.2", + "n3": "^1.3.4" }, "devDependencies": { "@types/node": "^12.12.14",
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/graph_view.js Wed Dec 04 00:09:15 2019 -0800 @@ -0,0 +1,202 @@ +// from /my/site/homepage/www/rdf/browse/graphView.js + + +import { SuffixLabels } from './suffixLabels.js'; + +const groupByRdfType = (graph) => { + const env = graph.store.rdf; + const rdfType = env.createNamedNode('rdf:type'); + const byType = new Map(); // type : subjs + const untyped = new Set(); // subjs + graph.quadStore.quads({}, (q) => { + let subjType = null; + graph.quadStore.quads({subject: q.subject, + predicate: rdfType}, + (q2) => { subjType = q2.object; }); + if (subjType){ + subjType = subjType.toString(); + if (!byType.has(subjType)) { + byType.set(subjType, new Set()); + } + byType.get(subjType).add(q.subject.toString()); + } else { + untyped.add(q.subject.toString()); + } + + }); + return {byType: byType, untyped: untyped}; +}; + +const graphView = (graph) => { + const env = graph.store.rdf; + + const labels = new SuffixLabels(); + graph.quadStore.quads({}, (q) => { + if (q.subject.interfaceName == "NamedNode") { labels.planDisplayForNode(q.subject); } + if (q.predicate.interfaceName == "NamedNode") { labels.planDisplayForNode(q.predicate); } + if (q.object.interfaceName == "NamedNode") { labels.planDisplayForNode(q.object); } + if (q.object.interfaceName == "Literal" && q.object.datatype) { labels.planDisplayForNode(env.createNamedNode(q.object.datatype)); } + }); + + const rdfNode = (n) => { + if (n.interfaceName == "Literal") { + let dtPart = ""; + if (n.datatype) { + dtPart = html` + ^^<span class="literalType"> + ${rdfNode(env.createNamedNode(n.datatype))} + </span>`; + } + return html`<span class="literal">${n.nominalValue}${dtPart}</span>`; + } + if (n.interfaceName == "NamedNode") { + let dn = labels.getLabelForNode(n); + if (dn.match(/XMLSchema#.*/)) { dn = dn.replace('XMLSchema#', 'xsd:'); } + if (dn.match(/rdf-schema#.*/)) { dn = dn.replace('rdf-schema#', 'rdfs:'); } + return html`<a class="graphUri" href="${n.toString()}">${dn}</a>`; + } + + return html`[${n.interfaceName} ${n.toNT()}]`; + } + + const objBlock = (obj) => { + return html` + <div class="object"> + ${rdfNode(obj)} <!-- indicate what source or graph said this stmt --> + </div> + `; + }; + + /// bunch of table rows + const predBlock = (subj, pred) => { + const objsSet = new Set(); + graph.quadStore.quads({ subject: subj, predicate: pred }, (q) => { + + if (q.object.length) { + console.log(q.object) + } + objsSet.add(q.object); + }); + const objs = Array.from(objsSet.values()); + objs.sort(); + return html` + <div class="predicate">${rdfNode(pred)} + <div> + ${objs.map(objBlock)} + </div> + </div> + `; + }; + + const {byType, untyped} = groupByRdfType(graph); + const typedSubjs = Array.from(byType.keys()); + typedSubjs.sort(); + + const untypedSubjs = Array.from(untyped.values()); + untypedSubjs.sort(); + + const subjBlock = (subj) => { + const subjNode = env.createNamedNode(subj); + const predsSet = new Set(); + graph.quadStore.quads({ subject: subjNode }, (q) => { + predsSet.add(q.predicate); + }); + const preds = Array.from(predsSet.values()); + preds.sort(); + return html` + <div class="subject">${rdfNode(subjNode)} + <!-- todo: special section for uri/type-and-icon/label/comment --> + <div> + ${preds.map((p) => { return predBlock(subjNode, p); })} + </div> + </div> + `; + }; + const byTypeBlock = (typeUri) => { + const subjs = Array.from(byType.get(typeUri)); + subjs.sort(); + + const graphCells = new Map(); // [subj, pred] : objs + const preds = new Set(); + + subjs.forEach((subj) => { + graph.quadStore.quads({subject: env.createNamedNode(subj)}, (q) => { + preds.add(q.predicate.toString()); + const cellKey = subj + '|||' + q.predicate.toString(); + if (!graphCells.has(cellKey)) { + graphCells.set(cellKey, new Set()); + } + graphCells.get(cellKey).add(q.object); + }); + }); + const predsList = Array.from(preds); + predsList.splice(predsList.indexOf('http://www.w3.org/1999/02/22-rdf-syntax-ns#type'), 1); + // also pull out label, which should be used on 1st column + predsList.sort(); + + const thead = () => { + const predColumnHead = (pred) => { + return html`<th>${rdfNode(env.createNamedNode(pred))}</th>`; + }; + return html` + <thead> + <tr> + <th></th> + ${predsList.map(predColumnHead)} + </tr> + </thead>`; + }; + + const instanceRow = (subj) => { + const cell = (pred) => { + const objs = graphCells.get(subj + '|||' + pred); + if (!objs) { + return html`<td></td>`; + } + const objsList = Array.from(objs); + objsList.sort(); + const draw = (obj) => { + return html`<div>${rdfNode(obj)}</div>` + }; + return html`<td>${objsList.map(draw)}</td>`; + }; + + return html` + <tr> + <td>${rdfNode(env.createNamedNode(subj))}</td> + ${predsList.map(cell)} + </tr> + `; + }; + + return html` + <div>[icon] ${rdfNode(env.createNamedNode(typeUri))} resources</div> +<div class="typeBlockScroll"> + <table class="typeBlock"> + ${thead()} + ${subjs.map(instanceRow)} + </table> +</div> + `; + }; + + return html` + <link rel="stylesheet" href="/rdf/browse/style.css"> + + <section> + <h2> + Current graph (<a href="${graph.events.url}">${graph.events.url}</a>) + </h2> + <div> + <!-- todo: graphs and provenance. + These statements are all in the + <span data-bind="html: $root.createCurie(graphUri())">...</span> graph.--> + </div> + ${typedSubjs.map(byTypeBlock)} + <div class="spoGrid"> + ${untypedSubjs.map(subjBlock)} + </div> + </section> + `; +} +export { graphView }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/json_ld_quads.ts Wed Dec 04 00:09:15 2019 -0800 @@ -0,0 +1,50 @@ +import { Quad, Node } from "./rdf_types"; +import * as jsonld from "jsonld"; + +function quadFromExpandedStatement(rdfEnv: any, subj: any, pred: string, obj: any, graphNode: any): Quad { + return { + subject: rdfEnv.createNamedNode(subj['@id']), + predicate: rdfEnv.createNamedNode(pred), + object: (obj['@id'] ? rdfEnv.createNamedNode(obj['@id']) : + rdfEnv.createLiteral(obj['@value'], obj['@language'], obj['@type'])), + graph: graphNode, + }; +} +function quadFromTypeStatement(rdfEnv: any, subj: any, obj: any, graphNode: any): Quad { + return { + subject: rdfEnv.createNamedNode(subj['@id']), + predicate: rdfEnv.createNamedNode('rdf:type'), + object: rdfEnv.createNamedNode(obj), + graph: graphNode, + }; +} + +export function eachJsonLdQuad(rdfEnv: any, jsonLdObj: object, onQuad: (Quad) => void, done: () => void) { + jsonld.expand(jsonLdObj, function onExpand(err, expanded) { + if (err) { + throw new Error(); + } + (expanded as [object]).forEach(function (g) { + var graph = g['@id']; + var graphNode = rdfEnv.createNamedNode(graph) as Node; + g['@graph'].forEach(function (subj) { + for (let pred in subj) { + if (pred.match(/^[^@]/)) { + subj[pred].forEach(function (obj) { + onQuad(quadFromExpandedStatement(rdfEnv, subj, pred, obj, graphNode)); + }); + } + else { + if (pred === "@type") { + subj[pred].forEach((obj) => { + onQuad(quadFromTypeStatement(rdfEnv, subj, obj, graphNode)); + }); + } + } + } + }); + }); + done(); + }); +} +;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/rdf_types.ts Wed Dec 04 00:09:15 2019 -0800 @@ -0,0 +1,7 @@ +export type Node = any; // todo +export type Quad = { + subject: Node; + predicate: Node; + object: Node; + graph: Node, +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/streamed-graph.ts Wed Dec 04 00:09:15 2019 -0800 @@ -0,0 +1,91 @@ + +// these are just for timebank- move them out +import '@polymer/polymer/lib/elements/dom-bind.js'; + + +import { PolymerElement, html } from '@polymer/polymer'; +import { customElement, property, computed } from '@polymer/decorators'; +import { render } from 'lit-html'; +// import { graphView } from '/rdf/browse/graphView.js'; +import { StreamedGraphClient } from './streamed_graph_client'; + +@customElement('streamed-graph') +class StreamedGraph extends PolymerElement { + @property({ type: String }) + url: string = ''; + + @property({ type: Object }) + graph: Object; + + @property({ type: Boolean }) + expanded: Boolean = false; + + @computed('expanded') + get expandAction() { + return this.expanded ? '-' : '+'; + } + + @property({ type: String }) + status: String = ''; + + sg: StreamedGraphClient; + graphView: Element; + graphViewDirty = true; + + static get template() { + return html` + <link rel="stylesheet" href="../src/streamed-graph.css"> + <div id="ui"> + <span class="expander"><button on-click="toggleExpand">{{expandAction}}</button></span> + StreamedGraph <a href="{{url}}">[source]</a>: + {{status}} + </div> + <div id="graphView">graph here + </div>`; + } + + ready() { + super.ready(); + this.graphView = this.shadowRoot.getElementById("graphView"); + } + + toggleExpand(ev) { + this.expanded = !this.expanded; + if (this.expanded) { + this.redrawGraph() + } else { + this.graphViewDirty = false; + render(null, this.graphView); + } + } + + redrawGraph() { + this.graphViewDirty = true; + requestAnimationFrame(this._redrawLater.bind(this)); + } + + _onUrl(url) { + // if (this.sg) { this.sg.close(); } + this.sg = new StreamedGraphClient( + url, + this.onGraphChanged.bind(this), + this.set.bind(this, 'status'), + [],//window.NS, + []); + } + + onGraphChanged() { + //this.graph = { version: this.graph.version + 1, graph: this.sg }; + if (this.expanded) { + this.redrawGraph(); + } + } + + _redrawLater() { + if (!this.graphViewDirty) return; + //render(graphView(this.graph.graph), this.graphView); + this.graphViewDirty = false; + } + + +} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/streamed_graph_client.ts Wed Dec 04 00:09:15 2019 -0800 @@ -0,0 +1,168 @@ +// from /my/site/homepage/www/rdf/streamed-graph.js + +import * as async from "async"; +import * as jsonld from "jsonld"; + +//import eachJsonLdQuad from "./json_ld_quads"; +import { Store, DataFactory } from "n3" + +/// <reference types="eventsource" /> +const EventSource = window.EventSource; + +export class StreamedGraphClient { + // onStatus: (msg: string) => void; + // onGraphChanged: () => void; + // store: Store; + // events: EventSource; + constructor( + eventsUrl: string, + onGraphChanged: () => void, + onStatus: (status: string) => void, + prefixes: Array<Record<string, string>>, + staticGraphUrls: Array<string>) { + console.log('new StreamedGraph', eventsUrl); + // holds a rdfstore.js store, which is synced to a server-side + // // store that sends patches over SSE + // this.onStatus = onStatus; + // this.onGraphChanged = onGraphChanged; + // this.onStatus('startup...'); + + // this.store = new Store({}); + + // // Object.keys(prefixes).forEach((prefix) => { + // // this.store.setPrefix(prefix, prefixes[prefix]); + // // }); + + // this.connect(eventsUrl); + // this.reconnectOnWake(); + + // staticGraphUrls.forEach((url) => { + // fetch(url).then((response) => response.text()) + // .then((body) => { + // // parse with n3, add to output + // }); + // }); + + } + + // reconnectOnWake() { + // // it's not this, which fires on every mouse-in on a browser window, and doesn't seem to work for screen-turned-back-on + // //window.addEventListener('focus', function() { this.connect(eventsUrl); }.bind(this)); + + // } + + // connect(eventsUrl: string) { + // // need to exit here if this obj has been replaced + + // this.onStatus('start connect...'); + // this.close(); + // if (this.events && this.events.readyState != EventSource.CLOSED) { + // this.onStatus('zombie'); + // throw new Error("zombie eventsource"); + // } + + + // this.events = new EventSource(eventsUrl); + + // this.events.addEventListener('error', (ev) => { + // // todo: this is piling up tons of retries and eventually multiple connections + // this.testEventUrl(eventsUrl); + // this.onStatus('connection lost- retrying'); + // setTimeout(() => { + // requestAnimationFrame(() => { + // this.connect(eventsUrl); + // }); + // }, 3000); + // }); + + // this.events.addEventListener('fullGraph', (ev) => { + // // this.updates.push({ type: 'fullGraph', data: ev.data }); + // // this.flushUpdates(); + // }); + + // this.events.addEventListener('patch', (ev) => { + // // this.updates.push({ type: 'patch', data: ev.data }); + // // this.flushUpdates(); + // }); + // this.onStatus('connecting...'); + // } + + // replaceFullGraph(jsonLdText: string, done: () => void) { + // // this.quadStore.clear(); + // // eachJsonLdQuad(this.store.rdf, JSON.parse(jsonLdText), + // // this.quadStore.add.bind(this.quadStore), function () { + // // done(); + // // }); + // // or this.store.insert([quad], quad.graph, function() {}); + // } + + + // patchGraph(patchJson: string, done: () => void) { + // var patch = JSON.parse(patchJson).patch; + + // // if (!this.store) { + // // throw new Error('store ' + this.store); + // // } + + // async.series([ + // // (done) => { + // // eachJsonLdQuad(this.store.rdf, patch.deletes, + // // this.quadStore.remove.bind(this.quadStore), done); + // // }, + // (done) => { + // // eachJsonLdQuad(this.store.rdf, patch.adds, + // // this.quadStore.add.bind(this.quadStore), done); + // }, + // /* seriesDone */ (done) => { + // done(); + // } + // ], done); + // } + // close() { + // if (this.events) { + // this.events.close(); + // } + // } + + // testEventUrl(eventsUrl: string): Promise<void> { + // return new Promise<void>((resolve, reject) => { + // this.onStatus('testing connection'); + // fetch(eventsUrl, { + // method: "HEAD", + // credentials: "include", + // }).then((value) => { + // if (value.status == 403) { + // reject(); + // return; + // } + // resolve(); + // }).catch((err) => { + // reject(); + // }); + // }); + // } + + // flushOneUpdate(update: Update, done: () => void) { + // if (update.type == 'fullGraph') { + // this.onStatus('sync- full graph update'); + // let onReplaced = () => { + // this.onStatus('synced'); + // this.onGraphChanged(); + // done(); + // }; + // this.replaceFullGraph(update.data, onReplaced); + // } else if (update.type == 'patch') { + // this.onStatus('sync- updating'); + // let onPatched = () => { + // this.onStatus('synced'); + // this.onGraphChanged(); + // done(); + // }; + // this.patchGraph(update.data, onPatched); + // } else { + // this.onStatus('sync- unknown update'); + // throw new Error(update.type); + // } + // } + +} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/suffixLabels.js Wed Dec 04 00:09:15 2019 -0800 @@ -0,0 +1,72 @@ +class SuffixLabels { + constructor() { + this.displayNodes = new Map(); // internal string : { label, link } + this.usedSuffixes = {usedBy: null, children: new Map()}; + } + + planDisplayForNode(node) { + const uri = node.nominalValue; + this._planDisplayForUri(uri); + }; + + _planDisplayForUri(uri) { + if (this.displayNodes.has(uri)) { + return; + } + + const segments = uri.split('/'); + let curs = this.usedSuffixes; + let label = null; + + for (let i = segments.length - 1; i >= 0; i--) { + const seg = segments[i]; + if (curs.usedBy && curs.usedBy != uri) { + this._prependClashingUri(curs); + } + + if (!curs.children.has(seg)) { + const child = {usedBy: null, children: new Map()}; + curs.children.set(seg, child); + + if (label === null ) { + label = SuffixLabels._tailSegments(uri, segments.length - i); + child.usedBy = uri; + } + } + curs = curs.children.get(seg); + } + this.displayNodes.set(uri, {label: label}); + } + + _prependClashingUri(curs) { + // Claim: When a clash is discovered, only 1 uri needs to + // change its length, and there will be only one child node to + // follow, and the clashing uri can be changed to prepend that + // one child (since we'll see it again if that one wasn't + // enough). + const clashNode = this.displayNodes.get(curs.usedBy); + const nextLeftSeg = curs.children.entries().next().value; + if (nextLeftSeg[1].usedBy) { + throw new Error("unexpected"); + } + + clashNode.label = nextLeftSeg[0] + '/' + clashNode.label; + nextLeftSeg[1].usedBy = curs.usedBy; + curs.usedBy = null; + + } + + getLabelForNode(node) { + return this.displayNodes.get(node.nominalValue).label; + } + + static _tailSegments(uri, n) { + let i = uri.length; + for (let rep = 0; rep < n; rep++) { + i = uri.lastIndexOf('/', i - 1); + } + return uri.substr(i + 1); + } +}; + +export { SuffixLabels }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/suffixLabels_test.html Wed Dec 04 00:09:15 2019 -0800 @@ -0,0 +1,91 @@ +<!doctype html> +<html> + <head> + <title>suffixLabels_test</title> + <meta charset="utf-8" /> + <link rel="shortcut icon" type="image/png" href="/lib/jasmine/3.4.0/jasmine_favicon.png"> + <link rel="stylesheet" type="text/css" href="/lib/jasmine/3.4.0/jasmine.css"> + + <script type="text/javascript" src="/lib/jasmine/3.4.0/jasmine.js"></script> + <script type="text/javascript" src="/lib/jasmine/3.4.0/jasmine-html.js"></script> + <script type="text/javascript" src="/lib/jasmine/3.4.0/boot.js"></script> + + <script type="module"> + + import { SuffixLabels } from './suffixLabels.js'; + + describe("_tailSegments", function() { + it("returns right amount", function() { + expect(SuffixLabels._tailSegments('http://foo/a/bb', 0)).toEqual(''); + expect(SuffixLabels._tailSegments('http://foo/a/bb', 1)).toEqual('bb'); + expect(SuffixLabels._tailSegments('http://foo/a/bb', 2)).toEqual('a/bb'); + expect(SuffixLabels._tailSegments('http://foo/a/bb', 3)).toEqual('foo/a/bb'); + expect(SuffixLabels._tailSegments('http://foo/a/bb', 4)).toEqual('/foo/a/bb'); + expect(SuffixLabels._tailSegments('http://foo/a/bb', 5)).toEqual('http://foo/a/bb'); + }); + it("ok with trailing slash", function() { + expect(SuffixLabels._tailSegments('http://foo/', 0)).toEqual(''); + expect(SuffixLabels._tailSegments('http://foo/', 1)).toEqual(''); + expect(SuffixLabels._tailSegments('http://foo/', 2)).toEqual('foo/'); + }); + }); + + describe("suffixLabels", function() { + const fakeNode = (uri) => { return { nominalValue: uri } }; + + it("returns whole url segments", function() { + const suf = new SuffixLabels(); + suf._planDisplayForUri('http://a/b/c/dd'); + suf._planDisplayForUri('http://a/b/c/ee'); + + expect(suf.getLabelForNode(fakeNode('http://a/b/c/dd'))).toEqual('dd'); + expect(suf.getLabelForNode(fakeNode('http://a/b/c/ee'))).toEqual('ee'); + }); + + it("doesn't treat a repeated uri as a name clash", function() { + const suf = new SuffixLabels(); + suf._planDisplayForUri('http://a/b/c'); + suf._planDisplayForUri('http://a/b/c'); + + expect(suf.getLabelForNode(fakeNode('http://a/b/c'))).toEqual('c'); + }); + + it("moves to two segments when needed", function() { + const suf = new SuffixLabels(); + suf._planDisplayForUri('http://a/b/c/d'); + suf._planDisplayForUri('http://a/b/f/d'); + + expect(suf.getLabelForNode(fakeNode('http://a/b/c/d'))).toEqual('c/d'); + expect(suf.getLabelForNode(fakeNode('http://a/b/f/d'))).toEqual('f/d'); + }); + + it("is ok with clashes at different segment positions", function() { + const suf = new SuffixLabels(); + suf._planDisplayForUri('http://z/z/z/a/b/c'); + suf._planDisplayForUri('http://a/b/c'); + + expect(suf.getLabelForNode(fakeNode('http://z/z/z/a/b/c'))).toEqual('z/a/b/c'); + expect(suf.getLabelForNode(fakeNode('http://a/b/c'))).toEqual('/a/b/c'); + }); + + it("uses appropriately long suffixes per uri", function() { + const suf = new SuffixLabels(); + suf._planDisplayForUri('http://a/b/c/d/e'); + suf._planDisplayForUri('http://a/b/f/d/e'); + suf._planDisplayForUri('http://a/b/c/g'); + suf._planDisplayForUri('http://a/z'); + + expect(suf.getLabelForNode(fakeNode('http://a/b/c/d/e'))).toEqual('c/d/e'); + expect(suf.getLabelForNode(fakeNode('http://a/b/f/d/e'))).toEqual('f/d/e'); + expect(suf.getLabelForNode(fakeNode('http://a/b/c/g'))).toEqual('g'); + expect(suf.getLabelForNode(fakeNode('http://a/z'))).toEqual('z'); + }); + + }); + </script> + + </head> + <body> + + </body> +</html>
--- a/tsconfig.json Tue Dec 03 21:48:36 2019 -0800 +++ b/tsconfig.json Wed Dec 04 00:09:15 2019 -0800 @@ -5,11 +5,13 @@ "target": "es6", "rootDirs": [ "node_modules", + "src" ], "esModuleInterop": true, "allowSyntheticDefaultImports": true }, "files": [ "./src/streamed-graph.ts", + "./src/streamed_graph_client.ts" ] }