Mercurial > code > home > repos > href

comparison lookup.py @ 30:e86642cf7393

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
style and requirements.txt cleanup
author drewp@bigasterisk.com
date Sun, 12 Jul 2020 13:33:54 -0700
parents 7c82ffbca5d0
children c23acc88324b
comparison
equal deleted inserted replaced
29:890584020372 30:e86642cf7393
1 #!bin/python
2 """ 1 """
3 serve some queries over bookmarks: 2 serve some queries over bookmarks:
4 3
5 /user 4 /user
6 /user/tag+tag+tag 5 /user/tag+tag+tag
7 6
8 and the add-bookmark stuff 7 and the add-bookmark stuff
9
10 """ 8 """
11 import pymongo, bottle, time, urllib.request, urllib.parse, urllib.error, datetime, json, logging 9 from collections import defaultdict
10 import datetime
11 import json
12 import logging
13 import time
14 import urllib.error
15 import urllib.parse
16 import urllib.request
17
18 from bottle import static_file
19 from dateutil.tz import tzlocal
20 import bottle
21 import pymongo
12 import requests 22 import requests
13 from collections import defaultdict 23
14 from urllib.parse import urlparse
15 from dateutil.tz import tzlocal
16 from bottle import static_file
17 from jadestache import Renderer 24 from jadestache import Renderer
25 from link import Links, NotFound
18 from pagetitle import PageTitle 26 from pagetitle import PageTitle
19 from link import Links, NotFound 27
20 db = pymongo.Connection('mongodb.default.svc.cluster.local', tz_aware=True)['href'] 28 db = pymongo.Connection('mongodb.default.svc.cluster.local',
29 tz_aware=True)['href']
21 pageTitle = PageTitle(db) 30 pageTitle = PageTitle(db)
22 links = Links(db) 31 links = Links(db)
23 renderer = Renderer(search_dirs=['template'], debug=bottle.DEBUG) 32 renderer = Renderer(search_dirs=['template'], debug=bottle.DEBUG)
24 log = logging.getLogger() 33 log = logging.getLogger()
25 34
35
26 def getLoginBar(): 36 def getLoginBar():
27 return requests.get("http://openid-proxy.default.svc.cluster.local:9023/_loginBar", 37 return requests.get(
28 headers={ 38 "http://openid-proxy.default.svc.cluster.local:9023/_loginBar",
29 "Cookie" : bottle.request.headers.get('cookie'), 39 headers={
30 'x-site': 'http://bigasterisk.com/openidProxySite/href', 40 "Cookie": bottle.request.headers.get('cookie'),
31 }).text 41 'x-site': 'http://bigasterisk.com/openidProxySite/href',
42 }).text
43
32 44
33 def getUser(): 45 def getUser():
34 agent = bottle.request.headers.get('x-foaf-agent', None) 46 agent = bottle.request.headers.get('x-foaf-agent', None)
35 username = db['user'].find_one({'_id':agent})['username'] if agent else None 47 username = db['user'].find_one({'_id': agent
48 })['username'] if agent else None
36 return username, agent 49 return username, agent
50
37 51
38 def siteRoot(): 52 def siteRoot():
39 try: 53 try:
40 return bottle.request.headers['x-site-root'].rstrip('/') 54 return bottle.request.headers['x-site-root'].rstrip('/')
41 except KeyError: 55 except KeyError:
42 log.warn(repr(bottle.request.__dict__)) 56 log.warn(repr(bottle.request.__dict__))
43 raise 57 raise
44 58
59
45 @bottle.route('/static/<path:path>') 60 @bottle.route('/static/<path:path>')
46 def server_static(path): 61 def server_static(path):
47 return static_file(path, root='static') 62 return static_file(path, root='static')
48 63
64
49 def recentLinks(user, tags, allowEdit): 65 def recentLinks(user, tags, allowEdit):
50 out = {'links':[]} 66 out = {'links': []}
51 t1 = time.time() 67 t1 = time.time()
52 spec = {'user':user} 68 spec = {'user': user}
53 if tags: 69 if tags:
54 spec['extracted.tags'] = {'$all' : tags} 70 spec['extracted.tags'] = {'$all': tags}
55 for doc in db['links'].find(spec, sort=[('t', -1)], limit=50): 71 for doc in db['links'].find(spec, sort=[('t', -1)], limit=50):
56 link = links.forDisplay(doc) 72 link = links.forDisplay(doc)
57 link['allowEdit'] = allowEdit 73 link['allowEdit'] = allowEdit
58 out['links'].append(link) 74 out['links'].append(link)
59 out['stats'] = {'queryTimeMs' : round((time.time() - t1) * 1000, 2)} 75 out['stats'] = {'queryTimeMs': round((time.time() - t1) * 1000, 2)}
60 return out 76 return out
77
61 78
62 def allTags(user, withTags=[]): 79 def allTags(user, withTags=[]):
63 """withTags limits results to other tags that have been used with 80 """withTags limits results to other tags that have been used with
64 those tags""" 81 those tags"""
65 withTags = set(withTags) 82 withTags = set(withTags)
66 count = defaultdict(lambda: 0) # tag : count 83 count = defaultdict(lambda: 0) # tag : count
67 for doc in db['links'].find({'user':user}, fields=['extracted.tags']): 84 for doc in db['links'].find({'user': user}, fields=['extracted.tags']):
68 docTags = set(doc.get('extracted', {}).get('tags', [])) 85 docTags = set(doc.get('extracted', {}).get('tags', []))
69 if withTags and not withTags.issubset(docTags): 86 if withTags and not withTags.issubset(docTags):
70 continue 87 continue
71 for t in docTags.difference(withTags): 88 for t in docTags.difference(withTags):
72 count[t] = count[t] + 1 89 count[t] = count[t] + 1
73 byFreq = [(n, t) for t,n in count.items()] 90 byFreq = [(n, t) for t, n in count.items()]
74 byFreq.sort(key=lambda n_t: (-n_t[0], n_t[1])) 91 byFreq.sort(key=lambda n_t: (-n_t[0], n_t[1]))
75 return [{'label': t, 'count': n} for n, t in byFreq] 92 return [{'label': t, 'count': n} for n, t in byFreq]
76 93
94
77 def renderWithTime(name, data): 95 def renderWithTime(name, data):
78 t1 = time.time() 96 t1 = time.time()
79 rendered = renderer.render_name(name, data) 97 rendered = renderer.render_name(name, data)
80 dt = (time.time() - t1) * 1000 98 dt = (time.time() - t1) * 1000
81 rendered = rendered.replace('TEMPLATETIME', "%.02f ms" % dt) 99 rendered = rendered.replace('TEMPLATETIME', "%.02f ms" % dt)
82 return rendered 100 return rendered
83 101
102
84 @bottle.route('/addLink') 103 @bottle.route('/addLink')
85 def addLink(): 104 def addLink():
86 out = { 105 out = {
87 'toRoot': siteRoot(), 106 'toRoot': siteRoot(),
88 'absRoot': siteRoot(), 107 'absRoot': siteRoot(),
89 'user': getUser()[0], 108 'user': getUser()[0],
90 'withKnockout': True, 109 'withKnockout': True,
91 'fillHrefJson': json.dumps(bottle.request.params.get('url', '')), 110 'fillHrefJson': json.dumps(bottle.request.params.get('url', '')),
92 'loginBar': getLoginBar(), 111 'loginBar': getLoginBar(),
93 } 112 }
94 return renderWithTime('add.jade', out) 113 return renderWithTime('add.jade', out)
114
95 115
96 @bottle.route('/addOverlay') 116 @bottle.route('/addOverlay')
97 def addOverlay(): 117 def addOverlay():
98 p = bottle.request.params 118 p = bottle.request.params
99 119
100 return "" 120 return ""
101 121
102 122
103 @bottle.route('/addLink/proposedUri') 123 @bottle.route('/addLink/proposedUri')
104 def proposedUri(): 124 def proposedUri():
105 uri = bottle.request.params.uri 125 uri = bottle.request.params.uri
106 user, _ = getUser() 126 user, _ = getUser()
107 127
108 try: 128 try:
109 prevDoc = links.find(uri, user) 129 prevDoc = links.find(uri, user)
110 except NotFound: 130 except NotFound:
111 prevDoc = None 131 prevDoc = None
112 132
113 return { 133 return {
114 'description': prevDoc['description'] if prevDoc else pageTitle.pageTitle(uri), 134 'description':
115 'tag' : prevDoc['tag'] if prevDoc else '', 135 prevDoc['description'] if prevDoc else pageTitle.pageTitle(uri),
116 'extended' : prevDoc['extended'] if prevDoc else '', 136 'tag':
117 'shareWith' : prevDoc.get('shareWith', []) if prevDoc else [], 137 prevDoc['tag'] if prevDoc else '',
138 'extended':
139 prevDoc['extended'] if prevDoc else '',
140 'shareWith':
141 prevDoc.get('shareWith', []) if prevDoc else [],
118 'suggestedTags': ['tag1', 'tag2'], 142 'suggestedTags': ['tag1', 'tag2'],
119 'existed': prevDoc is not None, 143 'existed':
120 } 144 prevDoc is not None,
121 145 }
122 if 0: 146
123 pass#proposal check existing links, get page title (stuff that in db), get tags from us and other serviecs. maybe the deferred ones ater 147
148 # proposal check existing links, get page title (stuff that in db), get tags from us and other serviecs. maybe the deferred ones ater
124 149
125 150
126 @bottle.route('/tags') 151 @bottle.route('/tags')
127 def tagFilterComplete(): 152 def tagFilterComplete():
128 params = bottle.request.params 153 params = bottle.request.params
134 159
135 out = [] 160 out = []
136 for t in allTags(params.user, withTags=haveTags): 161 for t in allTags(params.user, withTags=haveTags):
137 if partialTerm and partialTerm not in t['label']: 162 if partialTerm and partialTerm not in t['label']:
138 continue 163 continue
139 out.append({'id': t['label'], 164 out.append({
140 'text': "%s (%s%s)" % (t['label'], 165 'id':
141 t['count'], 166 t['label'],
142 " left" if haveTags else "")}) 167 'text':
143 168 "%s (%s%s)" % (t['label'], t['count'], " left" if haveTags else "")
144 return {'tags' : out} 169 })
145 170
171 return {'tags': out}
172
173
146 @bottle.route('/<user>/') 174 @bottle.route('/<user>/')
147 def userSlash(user): 175 def userSlash(user):
148 bottle.redirect(siteRoot() + "/%s" % urllib.parse.quote(user)) 176 bottle.redirect(siteRoot() + "/%s" % urllib.parse.quote(user))
177
149 178
150 @bottle.route('/<user>.json', method='GET') 179 @bottle.route('/<user>.json', method='GET')
151 def userAllJson(user): 180 def userAllJson(user):
152 data = recentLinks(user, [], allowEdit=getUser()[0] == user) 181 data = recentLinks(user, [], allowEdit=getUser()[0] == user)
153 data['toRoot'] = siteRoot() 182 data['toRoot'] = siteRoot()
154 return json.dumps(data) 183 return json.dumps(data)
155 184
185
156 @bottle.route('/<user>', method='GET') 186 @bottle.route('/<user>', method='GET')
157 def userAll(user): 187 def userAll(user):
158 return userLinks(user, "") 188 return userLinks(user, "")
159 189
160 190
161 @bottle.route('/<user>', method='POST') 191 @bottle.route('/<user>', method='POST')
162 def userAddLink(user): 192 def userAddLink(user):
163 if getUser()[0] != user: 193 if getUser()[0] != user:
164 raise ValueError("not logged in as %s" % user) 194 raise ValueError("not logged in as %s" % user)
165 print(repr(bottle.request.params.__dict__)) 195 print(repr(bottle.request.params.__dict__))
166 doc = links.fromPostdata(bottle.request.params, 196 doc = links.fromPostdata(bottle.request.params, user,
167 user,
168 datetime.datetime.now(tzlocal())) 197 datetime.datetime.now(tzlocal()))
169 links.insertOrUpdate(doc) 198 links.insertOrUpdate(doc)
170 199
171 print("notify about sharing to", repr(doc['shareWith'])) 200 print("notify about sharing to", repr(doc['shareWith']))
172 201
173 bottle.redirect(siteRoot() + '/' + user) 202 bottle.redirect(siteRoot() + '/' + user)
203
174 204
175 def parseTags(tagComponent): 205 def parseTags(tagComponent):
176 # the %20 is coming from davis.js, not me :( 206 # the %20 is coming from davis.js, not me :(
177 return [_f for _f in tagComponent.replace("%20", "+").split('+') if _f] 207 return [_f for _f in tagComponent.replace("%20", "+").split('+') if _f]
178 208
209
179 @bottle.route('/<user>/<tags:re:.*>.json') 210 @bottle.route('/<user>/<tags:re:.*>.json')
180 def userLinksJson(user, tags): 211 def userLinksJson(user, tags):
181 tags = parseTags(tags) 212 tags = parseTags(tags)
182 data = recentLinks(user, tags, allowEdit=getUser()[0] == user) 213 data = recentLinks(user, tags, allowEdit=getUser()[0] == user)
183 data['toRoot'] = siteRoot() 214 data['toRoot'] = siteRoot()
184 return json.dumps(data) 215 return json.dumps(data)
185 216
186 217
187 @bottle.route('/<user>/<tags>') 218 @bottle.route('/<user>/<tags>')
188 def userLinks(user, tags): 219 def userLinks(user, tags):
189 tags = parseTags(tags) 220 tags = parseTags(tags)
190 log.info('userLinks user=%r tags=%r', user, tags) 221 log.info('userLinks user=%r tags=%r', user, tags)
191 data = recentLinks(user, tags, allowEdit=getUser()[0] == user) 222 data = recentLinks(user, tags, allowEdit=getUser()[0] == user)
192 data['loginBar'] = getLoginBar() 223 data['loginBar'] = getLoginBar()
193 data['desc'] = ("%s's recent links" % user) + (" tagged %s" % (tags,) if tags else "") 224 data['desc'] = ("%s's recent links" % user) + (" tagged %s" %
225 (tags, ) if tags else "")
194 data['toRoot'] = siteRoot() 226 data['toRoot'] = siteRoot()
195 data['allTags'] = allTags(user) 227 data['allTags'] = allTags(user)
196 data['user'] = user 228 data['user'] = user
197 data['showPrivateData'] = (user == getUser()[0]) 229 data['showPrivateData'] = (user == getUser()[0])
198 230
199 data['pageTags'] = [{"word":t} for t in tags] 231 data['pageTags'] = [{"word": t} for t in tags]
200 data['stats']['template'] = 'TEMPLATETIME' 232 data['stats']['template'] = 'TEMPLATETIME'
201 return renderWithTime('links.jade', data) 233 return renderWithTime('links.jade', data)
234
202 235
203 @bottle.route('/templates') 236 @bottle.route('/templates')
204 def templates(): 237 def templates():
205 return json.dumps({'linklist': renderer.load_template("linklist.jade")}) 238 return json.dumps({'linklist': renderer.load_template("linklist.jade")})
206 239
240
207 @bottle.route('/') 241 @bottle.route('/')
208 def root(): 242 def root():
209 data = { 243 data = {
210 'loginBar': getLoginBar(), 244 'loginBar': getLoginBar(),
211 'toRoot': siteRoot(), 245 'toRoot': siteRoot(),
212 'stats': {'template': 'TEMPLATETIME'}, 246 'stats': {
213 'users': [{'user':doc['username']} for doc in db['user'].find()], 247 'template': 'TEMPLATETIME'
214 } 248 },
249 'users': [{
250 'user': doc['username']
251 } for doc in db['user'].find()],
252 }
215 return renderWithTime('index.jade', data) 253 return renderWithTime('index.jade', data)
216 254
255
217 if __name__ == '__main__': 256 if __name__ == '__main__':
218 logging.basicConfig(level=logging.INFO) 257 logging.basicConfig(level=logging.INFO)
219 bottle.run(server='gunicorn', host='0.0.0.0', port=10002, workers=4) 258 bottle.run(server='gunicorn', host='0.0.0.0', port=10002, workers=4)