zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 19:24:07 +01:00
Code Issues Projects Releases Packages Wiki Activity

o

Browse source
This commit is contained in:
Joseph Cheung 2023-02-23 15:29:52 +08:00
parent 81985d12c3
commit bb827e0bb5
3 changed files with 5 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
requirements-dev.txt
View file

@ -22,4 +22,4 @@ wlc==1.13
coloredlogs==15.0.1
requests
markdown
tiktoken
transformers

2
requirements.txt
View file

@ -18,4 +18,4 @@ typing_extensions==4.5.0
fasttext-predict==0.9.2.1
requests
markdown
tiktoken
transformers

5
searx/webapp.py
View file

@ -17,7 +17,7 @@ import requests
import markdown
import re
import datetime
import tiktoken
from transformers import GPT2TokenizerFast
from timeit import default_timer
from html import escape
@ -769,7 +769,8 @@ def search():
res['content'] = res['content'].replace("This Tweet was deleted by the Tweet author.","Deleted Tweet.")
tmp_prompt = res['title'] +'\n'+ res['content'] + '\n' + new_url +'\n'
if len( tiktoken.get_encoding("gpt2").encode(prompt + tmp_prompt +'\n' + "\n以上是问题 " + original_search_query + " 的搜索结果，删除与问题相关度低的内容，用简体中文分条总结简报，在文中用(链接)标注对应内容来源链接，不要把链接都放在最后。结果：") )<2990:
tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
if len( tokenizer(prompt + tmp_prompt +'\n' + "\n以上是问题 " + original_search_query + " 的搜索结果，删除与问题相关度低的内容，用简体中文分条总结简报，在文中用(链接)标注对应内容来源链接，不要把链接都放在最后。结果：")['input_ids'] )<2990:
prompt += tmp_prompt +'\n'
if prompt != "":
gpt = ""