From 092e15cc1db2faa6c2fffb6163c5147d386bf948 Mon Sep 17 00:00:00 2001
From: jibe-b <user701@orange.fr>
Date: Tue, 29 Mar 2016 11:59:16 +0200
Subject: [PATCH 1/2] [enh] Add BASE engine in category Science. Basic and
 advanced search.

---
 searx/engines/base.py | 122 ++++++++++++++++++++++++++++++++++++++++++
 searx/settings.yml    |   6 +++
 searx/webapp.py       |  24 +++++----
 3 files changed, 142 insertions(+), 10 deletions(-)
 create mode 100755 searx/engines/base.py

diff --git a/searx/engines/base.py b/searx/engines/base.py
new file mode 100755
index 000000000..66491d395
--- /dev/null
+++ b/searx/engines/base.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+
+"""
+ BASE (Scholar publications)
+
+ @website     https://base-search.net
+ @provide-api yes with authorization (https://api.base-search.net/)
+
+ @using-api   yes
+ @results     XML
+ @stable      ?
+ @parse       url, title, publishedDate, content
+ More info on api: http://base-search.net/about/download/base_interface.pdf
+"""
+
+from lxml import etree
+from urllib import urlencode
+from searx.utils import searx_useragent
+from cgi import escape
+from datetime import datetime
+import re
+
+
+categories = ['science']
+
+base_url = 'https://api.base-search.net/cgi-bin/BaseHttpSearchInterface.fcgi'\
+           + '?func=PerformSearch&{query}&boost=oa&hits={hits}&offset={offset}'
+
+# engine dependent config
+paging = True
+number_of_results = 10
+
+# shortcuts for advanced search
+shorcut_dict = {
+    # user-friendly keywords
+    'format:': 'dcformat:',
+    'author:': 'dccreator:',
+    'collection:': 'dccollection:',
+    'hdate:': 'dchdate:',
+    'contributor:': 'dccontributor:',
+    'coverage:': 'dccoverage:',
+    'date:': 'dcdate:',
+    'abstract:': 'dcdescription:',
+    'urls:': 'dcidentifier:',
+    'language:': 'dclanguage:',
+    'publisher:': 'dcpublisher:',
+    'relation:': 'dcrelation:',
+    'rights:': 'dcrights:',
+    'source:': 'dcsource:',
+    'subject:': 'dcsubject:',
+    'title:': 'dctitle:',
+    'type:': 'dcdctype:'
+}
+
+
+def request(query, params):
+    # replace shortcuts with API advanced search keywords
+    for key in shorcut_dict.keys():
+        query = re.sub(str(key), str(shorcut_dict[key]), query)
+
+    # basic search
+    offset = (params['pageno'] - 1) * number_of_results
+
+    string_args = dict(query=urlencode({'query': query}),
+                       offset=offset,
+                       hits=number_of_results)
+
+    params['url'] = base_url.format(**string_args)
+
+    params['headers']['User-Agent'] = searx_useragent()
+    return params
+
+
+def response(resp):
+    results = []
+
+    search_results = etree.XML(resp.content)
+
+    for entry in search_results.xpath('./result/doc'):
+        content = "No description available"
+
+        date = datetime.now()  # needed in case no dcdate is available for an item
+        for item in entry:
+            if item.attrib["name"] == "dchdate":
+                harvestDate = item.text
+
+            elif item.attrib["name"] == "dcdate":
+                date = item.text
+
+            elif item.attrib["name"] == "dctitle":
+                title = item.text
+
+            elif item.attrib["name"] == "dclink":
+                url = item.text
+
+            elif item.attrib["name"] == "dcdescription":
+                content = escape(item.text[:300])
+                if len(item.text) > 300:
+                    content += "..."
+
+# dates returned by the BASE API are not several formats
+        publishedDate = None
+        for date_format in ['%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%d', '%Y-%m', '%Y']:
+            try:
+                publishedDate = datetime.strptime(date, date_format)
+                break
+            except:
+                pass
+
+        if publishedDate is not None:
+            res_dict = {'url': url,
+                        'title': title,
+                        'publishedDate': publishedDate,
+                        'content': content}
+        else:
+            res_dict = {'url': url,
+                        'title': title,
+                        'content': content}
+
+        results.append(res_dict)
+
+    return results
diff --git a/searx/settings.yml b/searx/settings.yml
index 5ef74d955..994106bf8 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -34,9 +34,15 @@ outgoing: # communication with search engines
 #        - 1.1.1.2
 
 engines:
+<<<<<<< HEAD
   - name : arch linux wiki
     engine : archlinux
     shortcut : al
+=======
+  - name : base
+    engine : base
+    shortcut : bs
+>>>>>>> Add BASE engine in category Science. Basic and advanced request.
 
   - name : wikipedia
     engine : mediawiki
diff --git a/searx/webapp.py b/searx/webapp.py
index 66ba65a29..2b665d962 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -408,17 +408,21 @@ def index():
 
         # TODO, check if timezone is calculated right
         if 'publishedDate' in result:
-            result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z')
-            if result['publishedDate'].replace(tzinfo=None) >= datetime.now() - timedelta(days=1):
-                timedifference = datetime.now() - result['publishedDate'].replace(tzinfo=None)
-                minutes = int((timedifference.seconds / 60) % 60)
-                hours = int(timedifference.seconds / 60 / 60)
-                if hours == 0:
-                    result['publishedDate'] = gettext(u'{minutes} minute(s) ago').format(minutes=minutes)
-                else:
-                    result['publishedDate'] = gettext(u'{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes)  # noqa
+            try:  # test if publishedDate >= 1900 (datetime module bug)
+                result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z')
+            except ValueError:
+                result['publishedDate'] = None
             else:
-                result['publishedDate'] = format_date(result['publishedDate'])
+                if result['publishedDate'].replace(tzinfo=None) >= datetime.now() - timedelta(days=1):
+                    timedifference = datetime.now() - result['publishedDate'].replace(tzinfo=None)
+                    minutes = int((timedifference.seconds / 60) % 60)
+                    hours = int(timedifference.seconds / 60 / 60)
+                    if hours == 0:
+                        result['publishedDate'] = gettext(u'{minutes} minute(s) ago').format(minutes=minutes)
+                    else:
+                        result['publishedDate'] = gettext(u'{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes)  # noqa
+                else:
+                    result['publishedDate'] = format_date(result['publishedDate'])
 
     if search.request_data.get('format') == 'json':
         return Response(json.dumps({'query': search.query,

From 0b90cd42e238ac046e49626a59a02cafea3ba72c Mon Sep 17 00:00:00 2001
From: jibe-b <user701@orange.fr>
Date: Tue, 29 Mar 2016 16:18:46 +0200
Subject: [PATCH 2/2] minor update

---
 searx/settings.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/searx/settings.yml b/searx/settings.yml
index 994106bf8..462a0bcc2 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -34,15 +34,13 @@ outgoing: # communication with search engines
 #        - 1.1.1.2
 
 engines:
-<<<<<<< HEAD
   - name : arch linux wiki
     engine : archlinux
     shortcut : al
-=======
+
   - name : base
     engine : base
     shortcut : bs
->>>>>>> Add BASE engine in category Science. Basic and advanced request.
 
   - name : wikipedia
     engine : mediawiki