Merge pull request #158 from Cqoicebordel/Moar-Engines

Add 500px and Searchcode engines

searx/engines/500px.py

@@ -0,0 +1,57 @@
+## 500px (Images)
+#
+# @website     https://500px.com
+# @provide-api yes (https://developers.500px.com/)
+#
+# @using-api   no
+# @results     HTML
+# @stable      no (HTML can change)
+# @parse       url, title, thumbnail, img_src, content
+#
+# @todo        rewrite to api
+
+
+from urllib import urlencode
+from urlparse import urljoin
+from lxml import html
+
+# engine dependent config
+categories = ['images']
+paging = True
+
+# search-url
+base_url = 'https://500px.com'
+search_url = base_url+'/search?search?page={pageno}&type=photos&{query}'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(pageno=params['pageno'],
+                                      query=urlencode({'q': query}))
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    
+    dom = html.fromstring(resp.text)
+    
+    # parse results
+    for result in dom.xpath('//div[@class="photo"]'):
+        link = result.xpath('.//a')[0]
+        url = urljoin(base_url, link.attrib.get('href'))
+        title = result.xpath('.//div[@class="title"]//text()')[0]
+        img_src = link.xpath('.//img')[0].attrib['src']
+        content = result.xpath('.//div[@class="info"]//text()')[0]
+
+        # append result
+        results.append({'url': url,
+                        'title': title,
+                        'img_src': img_src,
+                        'content': content,
+                        'template': 'images.html'})
+
+    # return results
+    return results

searx/engines/searchcode_code.py

@@ -0,0 +1,65 @@
+## Searchcode (It)
+#
+# @website     https://searchcode.com/
+# @provide-api yes (https://searchcode.com/api/)
+#
+# @using-api   yes
+# @results     JSON
+# @stable      yes
+# @parse       url, title, content
+
+from urllib import urlencode
+from json import loads
+import cgi
+import re
+
+# engine dependent config
+categories = ['it']
+paging = True
+
+# search-url
+url = 'https://searchcode.com/'
+search_url = url+'api/codesearch_I/?{query}&p={pageno}'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(query=urlencode({'q': query}),
+                                      pageno=params['pageno']-1)
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    
+    search_results = loads(resp.text)
+
+    # parse results
+    for result in search_results['results']:
+        href = result['url']
+        title = "" + result['name'] + " - " + result['filename']
+        content = result['repo'] + "<br />"
+        
+        lines = dict()
+        for line, code in result['lines'].items():
+            lines[int(line)] = code
+
+        content = content + '<pre class="code-formatter"><table class="code">'
+        for line, code in sorted(lines.items()):
+            content = content + '<tr><td class="line-number" style="padding-right:5px;">' 
+            content = content + str(line) + '</td><td class="code-snippet">' 
+            # Replace every two spaces with ' &nbps;' to keep formatting while allowing the browser to break the line if necessary
+            content = content + cgi.escape(code).replace('\t', '    ').replace('  ', '&nbsp; ').replace('  ', ' &nbsp;') 
+            content = content + "</td></tr>"
+            
+        content = content + "</table></pre>"
+        
+        # append result
+        results.append({'url': href,
+                        'title': title,
+                        'content': content})
+
+    # return results
+    return results

searx/engines/searchcode_doc.py

@@ -0,0 +1,49 @@
+## Searchcode (It)
+#
+# @website     https://searchcode.com/
+# @provide-api yes (https://searchcode.com/api/)
+#
+# @using-api   yes
+# @results     JSON
+# @stable      yes
+# @parse       url, title, content
+
+from urllib import urlencode
+from json import loads
+
+# engine dependent config
+categories = ['it']
+paging = True
+
+# search-url
+url = 'https://searchcode.com/'
+search_url = url+'api/search_IV/?{query}&p={pageno}'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(query=urlencode({'q': query}),
+                                      pageno=params['pageno']-1)
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    
+    search_results = loads(resp.text)
+
+    # parse results
+    for result in search_results['results']:
+        href = result['url']
+        title = "[" + result['type'] + "] " + result['namespace'] + " " + result['name']
+        content = '<span class="highlight">[' + result['type'] + "] " + result['name'] + " " + result['synopsis'] + "</span><br />" + result['description']
+        
+        # append result
+        results.append({'url': href,
+                        'title': title,
+                        'content': content})
+
+    # return results
+    return results

searx/settings.yml

@@ -65,6 +65,10 @@ engines:
 #    categories : files
 #    shortcut : fc
     
+  - name : 500px
+    engine : 500px
+    shortcut : px
+
   - name : flickr
     engine : flickr
     categories : images
@@ -115,6 +119,14 @@ engines:
     engine : stackoverflow
     shortcut : st
     
+  - name : searchcode doc
+    engine : searchcode_doc
+    shortcut : scd
+    
+  - name : searchcode code
+    engine : searchcode_code
+    shortcut : scc
+
   - name : startpage
     engine : startpage
     shortcut : sp