mirror of
https://github.com/searxng/searxng
synced 2024-01-01 18:24:07 +00:00
3ff2ad939d
Some result items from core.ac.uk do not have an URL::
Traceback (most recent call last):
File "searx/search/processors/online.py", line 154, in search
search_results = self._search_basic(query, params)
File "searx/search/processors/online.py", line 142, in _search_basic
return self.engine.response(response)
File "SearXNG/searx/engines/core.py", line 73, in response
'url': source['urls'][0].replace('http://', 'https://', 1),
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
85 lines
2.1 KiB
Python
85 lines
2.1 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# lint: pylint
|
|
"""CORE (science)
|
|
|
|
"""
|
|
|
|
from json import loads
|
|
from datetime import datetime
|
|
from urllib.parse import urlencode
|
|
|
|
from searx.exceptions import SearxEngineAPIException
|
|
|
|
about = {
|
|
"website": 'https://core.ac.uk',
|
|
"wikidata_id": 'Q22661180',
|
|
"official_api_documentation": 'https://core.ac.uk/documentation/api/',
|
|
"use_official_api": True,
|
|
"require_api_key": True,
|
|
"results": 'JSON',
|
|
}
|
|
|
|
categories = ['science']
|
|
paging = True
|
|
nb_per_page = 10
|
|
|
|
api_key = 'unset'
|
|
|
|
base_url = 'https://core.ac.uk:443/api-v2/search/'
|
|
search_string = '{query}?page={page}&pageSize={nb_per_page}&apiKey={apikey}'
|
|
|
|
|
|
def request(query, params):
|
|
|
|
if api_key == 'unset':
|
|
raise SearxEngineAPIException('missing CORE API key')
|
|
|
|
search_path = search_string.format(
|
|
query=urlencode({'q': query}),
|
|
nb_per_page=nb_per_page,
|
|
page=params['pageno'],
|
|
apikey=api_key,
|
|
)
|
|
params['url'] = base_url + search_path
|
|
|
|
logger.debug("query_url --> %s", params['url'])
|
|
return params
|
|
|
|
|
|
def response(resp):
|
|
results = []
|
|
json_data = loads(resp.text)
|
|
|
|
for result in json_data['data']:
|
|
|
|
source = result['_source']
|
|
if not source['urls']:
|
|
continue
|
|
|
|
time = source['publishedDate'] or source['depositedDate']
|
|
if time:
|
|
date = datetime.fromtimestamp(time / 1000)
|
|
else:
|
|
date = None
|
|
|
|
metadata = []
|
|
if source['publisher'] and len(source['publisher']) > 3:
|
|
metadata.append(source['publisher'])
|
|
if source['topics']:
|
|
metadata.append(source['topics'][0])
|
|
if source['doi']:
|
|
metadata.append(source['doi'])
|
|
metadata = ' / '.join(metadata)
|
|
|
|
results.append(
|
|
{
|
|
'url': source['urls'][0].replace('http://', 'https://', 1),
|
|
'title': source['title'],
|
|
'content': source['description'],
|
|
'publishedDate': date,
|
|
'metadata': metadata,
|
|
}
|
|
)
|
|
|
|
return results
|