Mercurial > code > home > repos > homeauto
changeset 552:98384c3ccd33
update build and deps to use invoke and to use new lib layout, plus more stats collection
Ignore-this: 78f09ba8a1898fd5f79cfc260cd36185
| author | drewp@bigasterisk.com |
|---|---|
| date | Thu, 25 Apr 2019 23:38:47 -0700 |
| parents | 1faebc897c77 |
| children | 36e4304762ae |
| files | service/piNode/Dockerfile service/piNode/makefile service/piNode/piNode.py service/piNode/requirements.txt service/piNode/tasks.py |
| diffstat | 5 files changed, 77 insertions(+), 44 deletions(-) [+] |
line wrap: on
line diff
--- a/service/piNode/Dockerfile Thu Apr 25 23:36:44 2019 -0700 +++ b/service/piNode/Dockerfile Thu Apr 25 23:38:47 2019 -0700 @@ -3,15 +3,15 @@ # super slow build RUN pip install grpcio==1.18.0 +WORKDIR /opt + COPY requirements.txt ./ -RUN pip install -r requirements.txt +RUN pip install --index-url https://projects.bigasterisk.com/ --extra-index-url https://pypi.org/simple -r requirements.txt -ADD https://projects.bigasterisk.com/rdfdb/more.tgz ./ -RUN tar xvzf more.tgz - -ADD https://projects.bigasterisk.com/rdfdb/more2.tgz ./ -RUN tar xvzf more2.tgz +# when we go to py3: +## not sure why this doesn't work from inside requirements.txt +#RUN pip3 install -U 'https://github.com/drewp/cyclone/archive/python3.zip?v3' COPY *.py ./ COPY config/ ./config/
--- a/service/piNode/makefile Thu Apr 25 23:36:44 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,20 +0,0 @@ -build_image: gather_image_files - docker build --network host -t bang6:5000/pi_node_pi:latest . - docker push bang6:5000/pi_node_pi:latest - -gather_image_files: - (cd /my/proj/homeauto/lib; tar czf /my/site/projects/rdfdb/more.tgz *.py) - (cd /my/proj/homeauto/service/arduinoNode; tar czf /my/site/projects/rdfdb/more2.tgz static) - -shell: - docker run -it bang6:5000/pi_node_pi:latest /bin/bash - -local_run: - docker run -it -p 9059:9059 bang6:5000/pi_node_pi:latest python ./piNode.py -v - - -push_config: - docker run -it --net=host -v `pwd`/config:/opt/config bang6:5000/arduino_node python pushConfig.py pi/ - -redeploy: build_image - sudo /my/proj/ansible/playbook -l pi -t raspi_io_node
--- a/service/piNode/piNode.py Thu Apr 25 23:36:44 2019 -0700 +++ b/service/piNode/piNode.py Thu Apr 25 23:38:47 2019 -0700 @@ -1,5 +1,5 @@ from __future__ import division -import sys, logging, socket, json, time +import sys, logging, socket, json, time, pkg_resources import cyclone.web from cyclone.httpclient import fetch from rdflib import Namespace, URIRef, Literal, Graph, RDF, ConjunctiveGraph @@ -40,6 +40,12 @@ CTX = ROOM['pi/%s' % hostname] STATS = scales.collection('/root', + scales.PmfStat('configReread'), + scales.IntStat('pollException'), + scales.PmfStat('boardPoll'), + scales.PmfStat('sendOneshot'), + scales.PmfStat('outputStatements'), + ) def patchRandid(): """ @@ -84,7 +90,8 @@ if getattr(self, 'rereadLater', None): self.rereadLater.cancel() self.rereadLater = None - + + @STATS.configReread.time() def reread(self): self.rereadLater = None log.info('read config') @@ -135,10 +142,12 @@ def startPolling(self): task.LoopingCall(self._poll).start(.05) + @STATS.boardPoll.time() # not differentiating multiple boards here def _poll(self): try: self._pollMaybeError() except Exception: + STATS.pollException += 1 log.exception("During poll:") def _pollMaybeError(self): @@ -148,6 +157,8 @@ if (hasattr(i, 'pollPeriod') and self._lastPollTime.get(i.uri, 0) + i.pollPeriod > now): continue + #need something like: + # with i.pollTiming.time(): new = i.poll() pollTime[i.uri] = time.time() - now if isinstance(new, dict): # new style @@ -184,7 +195,8 @@ self.masterGraph.patch(Patch.fromDiff(inContext(prev, dev), inContext(new, dev))) self._statementsFromInputs[dev] = new - + + @STATS.sendOneshot.time() def _sendOneshot(self, oneshot): body = (' '.join('%s %s %s .' % (s.n3(), p.n3(), o.n3()) for s,p,o in oneshot)).encode('utf8') @@ -199,6 +211,7 @@ url, e.getErrorMessage()) d.addErrback(err) + @STATS.outputStatements.time() def outputStatements(self, stmts): unused = set(stmts) for dev in self._devs: @@ -306,9 +319,11 @@ masterGraph = PatchableGraph() config = Config(masterGraph, arg['--hub']) + static = pkg_resources.resource_filename('homeauto_anynode', 'static/') + reactor.listenTCP(9059, cyclone.web.Application([ (r"/()", cyclone.web.StaticFileHandler, { - "path": "static", "default_filename": "index.html"}), + "path": static, "default_filename": "index.html"}), (r'/static/(.*)', cyclone.web.StaticFileHandler, {"path": "static"}), (r'/stats/(.*)', StatsHandler, {'serverName': 'piNode'}), (r'/boards', Boards),
--- a/service/piNode/requirements.txt Thu Apr 25 23:36:44 2019 -0700 +++ b/service/piNode/requirements.txt Thu Apr 25 23:38:47 2019 -0700 @@ -1,20 +1,25 @@ +RPi.GPIO==0.6.2 cryptography -rdflib==4.2.2 -rdflib-jsonld==0.3 cyclone docopt -python-dateutil -w1thermsensor -service_identity +etcd3==0.8.1 +git+git://github.com/adafruit/Adafruit_Nokia_LCD +git+http://github.com/drewp/scales.git@448d59fb491b7631877528e7695a93553bfaaa93#egg=scales http://abyz.me.uk/rpi/pigpio/pigpio.zip -git+git://github.com/adafruit/Adafruit_Nokia_LCD -RPi.GPIO==0.6.2 influxdb==3.0.0 +pillow +pyOpenSSL +python-dateutil +rdflib-jsonld==0.3 +rdflib==4.2.2 rpi_ws281x==3.1.0 -pyOpenSSL -numpy -pillow -imageio==2.2.0 -https://projects.bigasterisk.com/rdfdb/rdfdb-0.8.0.tar.gz -etcd3==0.8.1 -git+http://github.com/drewp/scales.git@448d59fb491b7631877528e7695a93553bfaaa93#egg=scales +service_identity +w1thermsensor + +cycloneerr +devices_shared==0.3.0 +export_to_influxdb==0.1.0 +homeauto_anynode==0.5.0 +patchablegraph==0.5.0 +rdfdb==0.8.0 +standardservice==0.4.0
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/piNode/tasks.py Thu Apr 25 23:38:47 2019 -0700 @@ -0,0 +1,33 @@ +from invoke import task + +JOB = 'piNode' +PORT = 9059 +TAG = f'bang6:5000/{JOB.lower()}_pi:latest' +ANSIBLE_TAG = 'raspi_io_node' + +@task +def build_image(ctx): + ctx.run(f'docker build --network=host -t {TAG} .') + +@task(pre=[build_image]) +def push_image(ctx): + ctx.run(f'docker push {TAG}') + +# (cd /my/proj/homeauto/service/arduinoNode; tar czf /my/site/projects/rdfdb/more2.tgz static) + +@task(pre=[build_image]) +def shell(ctx): + ctx.run(f'docker run --name={JOB}_shell --rm -it --cap-add SYS_PTRACE --net=host {TAG} /bin/bash', pty=True) + + +@task(pre=[build_image]) +def local_run(ctx): + ctx.run(f'docker run --name={JOB}_local --rm -it {TAG} python ./piNode.py -v', pty=True) + +@task(pre=[push_image]) +def redeploy(ctx): + ctx.run(f'sudo /my/proj/ansible/playbook -l pi -t {ANSIBLE_TAG}') + +@task +def push_config(ctx): + ctx.run(f'docker run --rm --net=host -v `pwd`/config:/opt/config bang6:5000/arduino_node python pushConfig.py pi/')