From 96ea355a1f96078b2894f702f293304a83e72146 Mon Sep 17 00:00:00 2001 From: ta Date: Sun, 14 Aug 2022 08:39:41 +0700 Subject: [PATCH 1/4] add twitter engine --- searx/engines/twitter.py | 63 ++++++++++++++++++++++++++++++++++++++++ searx/settings.yml | 5 ++++ 2 files changed, 68 insertions(+) create mode 100644 searx/engines/twitter.py diff --git a/searx/engines/twitter.py b/searx/engines/twitter.py new file mode 100644 index 000000000..d5d276e33 --- /dev/null +++ b/searx/engines/twitter.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Twitter (microblogging platform)""" + +from json import loads +from urllib.parse import urlencode +from datetime import datetime + +about = { + "website": 'https://twitter.com', + "wikidata_id": None, + "official_api_documentation": 'https://developer.twitter.com/en/docs/twitter-api', + "use_official_api": True, + "require_api_key": False, + "results": 'JSON', +} + +categories = ['social media'] + +url = "https://api.twitter.com" +search_url = ( + "{url}/2/search/adaptive.json?{query}&tweet_mode=extended&query_source=typed_query&pc=1&spelling_corrections=1" +) + + +def request(query, params): + params['url'] = search_url.format(url=url, query=urlencode({'q': query})) + + params['headers'] = { + # https://github.com/zedeus/nitter/blob/5f31e86e0e8578377fa7d5aeb9631bbb2d35ef1e/src/consts.nim#L5 + 'Authorization': ( + "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKb" + "T3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" + ) + } + + return params + + +def response(resp): + results = [] + + json_res = loads(resp.text)['globalObjects'] + + for tweet in json_res['tweets'].values(): + text = tweet['full_text'] + display = tweet['display_text_range'] + + results.append( + { + 'url': 'https://twitter.com/i/web/status/' + tweet['id_str'], + 'title': (text[:40] + '...') if len(text) > 40 else text, + 'content': text[display[0] : display[1]], + 'publishedDate': datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y'), + } + ) + + for user in json_res['users'].values(): + results.append( + {'title': user['name'], 'content': user['description'], 'url': 'https://twitter.com/' + user['screen_name']} + ) + + return results diff --git a/searx/settings.yml b/searx/settings.yml index 6cae30c48..1c9ba33c1 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -1421,6 +1421,11 @@ engines: # - 2000 # - 5000 + - name: twitter + shortcut: tw + engine: twitter + disabled: true + # maybe in a fun category # - name: uncyclopedia # engine: mediawiki From b6fd7cd571c52924f98e907b529cbc11865aba44 Mon Sep 17 00:00:00 2001 From: ta Date: Wed, 17 Aug 2022 14:25:22 +0700 Subject: [PATCH 2/4] add thumbnail to results if available --- searx/engines/twitter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/searx/engines/twitter.py b/searx/engines/twitter.py index d5d276e33..15d92c0e9 100644 --- a/searx/engines/twitter.py +++ b/searx/engines/twitter.py @@ -46,11 +46,16 @@ def response(resp): text = tweet['full_text'] display = tweet['display_text_range'] + img_src = tweet.get('extended_entities', {}).get('media', [{}])[0].get('media_url_https') + if img_src: + img_src += "?name=thumb" + results.append( { 'url': 'https://twitter.com/i/web/status/' + tweet['id_str'], 'title': (text[:40] + '...') if len(text) > 40 else text, 'content': text[display[0] : display[1]], + 'img_src': img_src, 'publishedDate': datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y'), } ) From c8acd4a3b6a664e0080ba6d5aebe0466dce54557 Mon Sep 17 00:00:00 2001 From: ta Date: Wed, 17 Aug 2022 14:30:59 +0700 Subject: [PATCH 3/4] add profile image to user results --- searx/engines/twitter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/searx/engines/twitter.py b/searx/engines/twitter.py index 15d92c0e9..b6eef0461 100644 --- a/searx/engines/twitter.py +++ b/searx/engines/twitter.py @@ -62,7 +62,12 @@ def response(resp): for user in json_res['users'].values(): results.append( - {'title': user['name'], 'content': user['description'], 'url': 'https://twitter.com/' + user['screen_name']} + { + 'title': user['name'], + 'content': user['description'], + 'url': 'https://twitter.com/' + user['screen_name'], + 'img_src': user['profile_image_url_https'], + } ) return results From 05851978cfecb987852d0c6244df7cecc9bfb7b8 Mon Sep 17 00:00:00 2001 From: ta Date: Wed, 17 Aug 2022 19:45:42 +0700 Subject: [PATCH 4/4] add explanation of token --- searx/engines/twitter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/searx/engines/twitter.py b/searx/engines/twitter.py index b6eef0461..3ebe34be1 100644 --- a/searx/engines/twitter.py +++ b/searx/engines/twitter.py @@ -27,6 +27,8 @@ def request(query, params): params['url'] = search_url.format(url=url, query=urlencode({'q': query})) params['headers'] = { + # This token is used in the Twitter web interface (twitter.com). Without this header, the API doesn't work. + # The value of the token has never changed (or maybe once a long time ago). # https://github.com/zedeus/nitter/blob/5f31e86e0e8578377fa7d5aeb9631bbb2d35ef1e/src/consts.nim#L5 'Authorization': ( "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKb"