Mercurial > code > home > repos > homeauto
changeset 1245:4c123099e5b6
collector build improvements; stats and logging
Ignore-this: 22fbbb2cde6a6bdbd9d0017f1f157a41
darcs-hash:3bcbb4fa359c1165e1c900ce0264869567e1d1fa
| author | drewp <drewp@bigasterisk.com> |
|---|---|
| date | Thu, 18 Apr 2019 09:15:39 -0700 |
| parents | cda481e4b77b |
| children | f18b95f81d43 |
| files | service/collector/Dockerfile service/collector/makefile service/collector/requirements.txt service/collector/sse_collector.py |
| diffstat | 4 files changed, 25 insertions(+), 11 deletions(-) [+] |
line wrap: on
line diff
--- a/service/collector/Dockerfile Thu Apr 18 09:15:22 2019 -0700 +++ b/service/collector/Dockerfile Thu Apr 18 09:15:39 2019 -0700 @@ -4,6 +4,7 @@ COPY requirements.txt ./ RUN pip install -r requirements.txt +RUN pip install py-spy COPY twisted_sse_demo ./twisted_sse_demo COPY *.py req* ./
--- a/service/collector/makefile Thu Apr 18 09:15:22 2019 -0700 +++ b/service/collector/makefile Thu Apr 18 09:15:39 2019 -0700 @@ -9,22 +9,33 @@ cp -a Dockerfile ../../lib/*.py ../../lib/twisted_sse_demo *.py req* tmp_ctx docker build --network=host -t ${TAG} tmp_ctx docker push ${TAG} - rm -r tmp_ctx shell: docker run --rm -it --cap-add SYS_PTRACE --net=host ${TAG} /bin/bash -local_run: +local_run: build_image docker run --rm -it -p ${PORT}:${PORT} \ - -v `pwd`:/mnt \ --net=host \ ${TAG} \ - python /mnt/sse_collector.py -v + python sse_collector.py -v -local_run_strace: +local_run_strace: build_image docker run --rm -it -p ${PORT}:${PORT} \ -v `pwd`:/mnt \ --net=host \ --cap-add SYS_PTRACE \ ${TAG} \ strace -f -tts 200 python /mnt/sse_collector.py -v + +local_run_pyspy: build_image + docker run --rm -it -p ${PORT}:${PORT} \ + -v `pwd`:/mnt \ + --net=host \ + --cap-add SYS_PTRACE \ + ${TAG} \ + py-spy -- python /mnt/sse_collector.py + + +redeploy: build_image + supervisorctl restart sse_collector_9072 +
--- a/service/collector/requirements.txt Thu Apr 18 09:15:22 2019 -0700 +++ b/service/collector/requirements.txt Thu Apr 18 09:15:39 2019 -0700 @@ -13,4 +13,4 @@ git+http://github.com/drewp/rdflib-jsonld.git@0a560c9f1aa7c7bbb80fea389e1f5fa51d1287f8#egg=rdflib_jsonld git+http://github.com/drewp/scales.git@448d59fb491b7631877528e7695a93553bfaaa93#egg=scales -https://projects.bigasterisk.com/rdfdb/rdfdb-0.3.0.tar.gz +https://projects.bigasterisk.com/rdfdb/rdfdb-0.8.0.tar.gz
--- a/service/collector/sse_collector.py Thu Apr 18 09:15:22 2019 -0700 +++ b/service/collector/sse_collector.py Thu Apr 18 09:15:39 2019 -0700 @@ -12,7 +12,7 @@ no_setup() import sys, logging, collections, json, time -from twisted.internet import reactor +from twisted.internet import reactor, defer import cyclone.web, cyclone.sse from rdflib import URIRef, Namespace from docopt import docopt @@ -260,6 +260,7 @@ for source in sources: if source not in self.clients and source != COLLECTOR: + log.debug('connect to patch source %s', source) self._localStatements.setSourceState(source, ROOM['connect']) self.clients[source] = ReconnectingPatchSource( source, listener=lambda p, fullGraph, source=source: self._onPatch( @@ -268,9 +269,7 @@ def removeSseHandler(self, handler): log.info('removeSseHandler %r', handler) - self.statements.discardHandler(handler) - for source in self._sourcesForHandler(handler): for otherHandler in self.handlers: if (otherHandler != handler and @@ -347,11 +346,13 @@ if arg['-v']: import twisted.python.log twisted.python.log.startLogging(sys.stdout) - log.setLevel(logging.INFO) + log.setLevel(logging.DEBUG) + defer.setDebugging(True) graphClients = GraphClients() - + #exporter = InfluxExporter(... to export some stats values + reactor.listenTCP( 9072, cyclone.web.Application( @@ -359,6 +360,7 @@ (r'/', Root), (r'/state', State), (r'/graph/(.*)', SomeGraph), + (r'/stats/(.*)', StatsHandler, {'serverName': 'collector'}), ], graphClients=graphClients), interface='::')