yannic/stacosys
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

manage imap/smtp directly. remove srmail dependency

Browse source
This commit is contained in:
Yax 2020-03-29 15:43:13 +02:00
parent 3281dd1aae
commit 7f5ea9c4d3
16 changed files with 436 additions and 208 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
config.ini
View file

@ -1,3 +1,4 @@
;
; Default configuration ; Default configuration
[main] [main]
lang = fr lang = fr
@ -5,18 +6,24 @@ db_url = sqlite:///db.sqlite
newcomment_polling = 60 newcomment_polling = 60
[http] [http]
root_url = http://localhost:8100 host = 127.0.0.1
host = 0.0.0.0
port = 8100 port = 8100
[security]
salt = BRRJRqXgGpXWrgTidBPcixIThHpDuKc0
secret = Uqca5Kc8xuU6THz9
[rss] [rss]
proto = http proto = https
file = comments.xml file = comments.xml
[mail] [imap]
fetch_polling = 30 polling = 120
mailer_url = http://localhost:8000 host = mail.gandi.net
ssl = false
port = 993
login = blog@mydomain.com
password = MYPASSWORD
[smtp]
host = mail.gandi.net
starttls = true
port = 587
login = blog@mydomain.com
password = MYPASSWORD

18
poetry.lock generated
View file

@ -363,6 +363,17 @@ urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26"
security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"] security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"]
[[package]]
category = "dev"
description = "a python refactoring library..."
name = "rope"
optional = false
python-versions = "*"
version = "0.16.0"
[package.extras]
dev = ["pytest"]
[[package]] [[package]]
category = "main" category = "main"
description = "Python 2 and 3 compatibility utilities" description = "Python 2 and 3 compatibility utilities"
@ -449,7 +460,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["pathlib2", "contextlib2", "unittest2"] testing = ["pathlib2", "contextlib2", "unittest2"]
[metadata] [metadata]
content-hash = "6270dbd1455ca926e89cdd874748cf8cee18c2ef5a10a05b6dc7f7100e2482d5" content-hash = "d698fc06cf58f4d228449cb76f48e8d739c323abd535427746d70dbbcaa4924c"
python-versions = "^3.7" python-versions = "^3.7"
[metadata.files] [metadata.files]
@ -618,6 +629,11 @@ requests = [
{file = "requests-2.22.0-py2.py3-none-any.whl", hash = "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"}, {file = "requests-2.22.0-py2.py3-none-any.whl", hash = "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"},
{file = "requests-2.22.0.tar.gz", hash = "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4"}, {file = "requests-2.22.0.tar.gz", hash = "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4"},
] ]
rope = [
{file = "rope-0.16.0-py2-none-any.whl", hash = "sha256:ae1fa2fd56f64f4cc9be46493ce54bed0dd12dee03980c61a4393d89d84029ad"},
{file = "rope-0.16.0-py3-none-any.whl", hash = "sha256:52423a7eebb5306a6d63bdc91a7c657db51ac9babfb8341c9a1440831ecf3203"},
{file = "rope-0.16.0.tar.gz", hash = "sha256:d2830142c2e046f5fc26a022fe680675b6f48f81c7fc1f03a950706e746e9dfe"},
]
six = [ six = [
{file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"}, {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"},
{file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"}, {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"},

1
pyproject.toml
View file

@ -19,6 +19,7 @@ requests = "^2.22.0"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
pytest = "^5.2" pytest = "^5.2"
black = {version = "^19.10b0", allow-prereleases = true} black = {version = "^19.10b0", allow-prereleases = true}
rope = "^0.16.0"
[build-system] [build-system]
requires = ["poetry>=0.12"] requires = ["poetry>=0.12"]

41
stacosys/conf/config.py
View file

@ -4,23 +4,30 @@
import profig import profig
# constants # constants
FLASK_APP = "flask.app" FLASK_APP = 'flask.app'
DB_URL = "main.db_url" DB_URL = 'main.db_url'
LANG = "main.lang" LANG = 'main.lang'
COMMENT_POLLING = 'main.newcomment_polling'
HTTP_HOST = "http.host" HTTP_HOST = 'http.host'
HTTP_PORT = "http.port" HTTP_PORT = 'http.port'
SECURITY_SALT = "security.salt" RSS_PROTO = 'rss.proto'
SECURITY_SECRET = "security.secret" RSS_FILE = 'rss.file'
RSS_PROTO = "rss.proto" IMAP_POLLING = 'imap.polling'
RSS_FILE = "rss.file" IMAP_SSL = 'imap.ssl'
IMAP_HOST = 'imap.host'
IMAP_PORT = 'imap.port'
IMAP_LOGIN = 'imap.login'
IMAP_PASSWORD = 'imap.password'
MAIL_POLLING = "mail.fetch_polling" SMTP_STARTTLS = 'smtp.starttls'
COMMENT_POLLING = "main.newcomment_polling" SMTP_HOST = 'smtp.host'
MAILER_URL = "mail.mailer_url" SMTP_PORT = 'smtp.port'
SMTP_LOGIN = 'smtp.login'
SMTP_PASSWORD = 'smtp.password'
# variable # variable
@ -38,16 +45,12 @@ def get(key):
return params[key] return params[key]
def getInt(key): def get_int(key):
return int(params[key]) return int(params[key])
def _str2bool(v): def get_bool(key):
return v.lower() in ("yes", "true", "t", "1") return params[key].lower() in ('yes', 'true', '1')
def getBool(key):
return _str2bool(params[key])
def flaskapp(): def flaskapp():

94
stacosys/core/cron.py
View file

@ -2,21 +2,21 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging import logging
from datetime import datetime
import time
import re import re
from core import mailer import time
from datetime import datetime
from core import mailer, rss
from core.templater import get_template from core.templater import get_template
from core import rss from model.comment import Comment, Site
from model.comment import Comment from model.email import Email
from model.comment import Site
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def cron(func): def cron(func):
def wrapper(): def wrapper():
logger.debug("execute CRON " + func.__name__) logger.debug('execute CRON ' + func.__name__)
func() func()
return wrapper return wrapper
@ -26,10 +26,10 @@ def cron(func):
def fetch_mail_answers(): def fetch_mail_answers():
for msg in mailer.fetch(): for msg in mailer.fetch():
if re.search(r".*STACOSYS.*\[(\d+)\:(\w+)\]", msg["subject"], re.DOTALL): if re.search(r'.*STACOSYS.*\[(\d+)\:(\w+)\]', msg.subject, re.DOTALL):
full_msg = mailer.get(msg["id"]) if full_msg and _reply_comment_email(msg):
if full_msg and reply_comment_email(full_msg['email']): mailer.delete(msg.id)
mailer.delete(msg["id"])
@cron @cron
def submit_new_comment(): def submit_new_comment():
@ -37,42 +37,36 @@ def submit_new_comment():
for comment in Comment.select().where(Comment.notified.is_null()): for comment in Comment.select().where(Comment.notified.is_null()):
comment_list = ( comment_list = (
"author: %s" % comment.author_name, 'author: %s' % comment.author_name,
"site: %s" % comment.author_site, 'site: %s' % comment.author_site,
"date: %s" % comment.created, 'date: %s' % comment.created,
"url: %s" % comment.url, 'url: %s' % comment.url,
"", '',
"%s" % comment.content, '%s' % comment.content,
"", '',
) )
comment_text = "\n".join(comment_list) comment_text = '\n'.join(comment_list)
email_body = get_template("new_comment").render( email_body = get_template('new_comment').render(
url=comment.url, comment=comment_text url=comment.url, comment=comment_text
) )
# send email # send email
site = Site.get(Site.id == comment.site) site = Site.get(Site.id == comment.site)
subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token) subject = 'STACOSYS %s: [%d:%s]' % (site.name, comment.id, site.token)
mailer.send(site.admin_email, subject, email_body) if mailer.send(site.admin_email, subject, email_body):
logger.debug("new comment processed ") logger.debug('new comment processed ')
# notify site admin and save notification datetime # notify site admin and save notification datetime
comment.notify_site_admin() comment.notify_site_admin()
else:
logger.warn('rescheduled. send mail failure ' + subject)
def reply_comment_email(data): def _reply_comment_email(email):
from_email = data["from"] m = re.search(r'\[(\d+)\:(\w+)\]', email.subject)
subject = data["subject"]
message = ""
for part in data["parts"]:
if part["content-type"] == "text/plain":
message = part["content"]
break
m = re.search(r"\[(\d+)\:(\w+)\]", subject)
if not m: if not m:
logger.warn("ignore corrupted email. No token %s" % subject) logger.warn('ignore corrupted email. No token %s' % email.subject)
return return
comment_id = int(m.group(1)) comment_id = int(m.group(1))
token = m.group(2) token = m.group(2)
@ -81,37 +75,39 @@ def reply_comment_email(data):
try: try:
comment = Comment.select().where(Comment.id == comment_id).get() comment = Comment.select().where(Comment.id == comment_id).get()
except: except:
logger.warn("unknown comment %d" % comment_id) logger.warn('unknown comment %d' % comment_id)
return True return True
if comment.published: if comment.published:
logger.warn("ignore already published email. token %d" % comment_id) logger.warn('ignore already published email. token %d' % comment_id)
return return
if comment.site.token != token: if comment.site.token != token:
logger.warn("ignore corrupted email. Unknown token %d" % comment_id) logger.warn('ignore corrupted email. Unknown token %d' % comment_id)
return return
if not message: if not email.content:
logger.warn("ignore empty email") logger.warn('ignore empty email')
return return
# safe logic: no answer or unknown answer is a go for publishing # safe logic: no answer or unknown answer is a go for publishing
if message[:2].upper() in ("NO"): if email.content[:2].upper() in ('NO'):
logger.info("discard comment: %d" % comment_id) logger.info('discard comment: %d' % comment_id)
comment.delete_instance() comment.delete_instance()
email_body = get_template("drop_comment").render(original=message) new_email_body = get_template('drop_comment').render(original=email.content)
mailer.send(from_email, "Re: " + subject, email_body) if not mailer.send(email.from_addr, 'Re: ' + email.subject, new_email_body):
logger.warn('minor failure. cannot send rejection mail ' + email.subject)
else: else:
# save publishing datetime # save publishing datetime
comment.publish() comment.publish()
logger.info("commit comment: %d" % comment_id) logger.info('commit comment: %d' % comment_id)
# rebuild RSS # rebuild RSS
rss.generate_site(token) rss.generate_site(token)
# send approval confirmation email to admin # send approval confirmation email to admin
email_body = get_template("approve_comment").render(original=message) new_email_body = get_template('approve_comment').render(original=email.content)
mailer.send(from_email, "Re: " + subject, email_body) if not mailer.send(email.from_addr, 'Re: ' + email.subject, new_email_body):
logger.warn('minor failure. cannot send approval email ' + email.subject)
return True return True

3
stacosys/core/database.py
View file

@ -1,9 +1,10 @@
#!/usr/bin/python #!/usr/bin/python
# -*- coding: UTF-8 -*- # -*- coding: UTF-8 -*-
from conf import config
from playhouse.db_url import connect from playhouse.db_url import connect
from conf import config
def get_db(): def get_db():
return connect(config.get(config.DB_URL)) return connect(config.get(config.DB_URL))

152
stacosys/core/imap.py Executable file
View file

@ -0,0 +1,152 @@
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import base64
import datetime
import email
import imaplib
import logging
import re
filename_re = re.compile('filename="(.+)"|filename=([^;\n\r"\']+)', re.I | re.S)
class Mailbox(object):
def __init__(self, host, port, ssl, login, password):
self.logger = logging.getLogger(__name__)
self.host = host
self.port = port
self.ssl = ssl
self.login = login
self.password = password
def __enter__(self):
if self.ssl:
self.imap = imaplib.IMAP4_SSL(self.host, self.port)
else:
self.imap = imaplib.IMAP4(self.host, self.port)
self.imap.login(self.login, self.password)
return self
def __exit__(self, type, value, traceback):
self.imap.close()
self.imap.logout()
def get_count(self):
self.imap.select('Inbox')
_, data = self.imap.search(None, 'ALL')
return sum(1 for num in data[0].split())
def fetch_raw_message(self, num):
self.imap.select('Inbox')
_, data = self.imap.fetch(str(num), '(RFC822)')
email_msg = email.message_from_bytes(data[0][1])
return email_msg
def fetch_message(self, num):
raw_msg = self.fetch_raw_message(num)
msg = {}
msg['encoding'] = 'UTF-8'
msg['index'] = num
dt = parse_date(raw_msg['Date']).strftime('%Y-%m-%d %H:%M:%S')
msg['datetime'] = dt
msg['from'] = raw_msg['From']
msg['to'] = raw_msg['To']
subject = email_nonascii_to_uft8(raw_msg['Subject'])
msg['subject'] = subject
parts = []
attachments = []
for part in raw_msg.walk():
if part.is_multipart():
continue
content_disposition = part.get('Content-Disposition', None)
if content_disposition:
# we have attachment
r = filename_re.findall(content_disposition)
if r:
filename = sorted(r[0])[1]
else:
filename = 'undefined'
content = base64.b64encode(part.get_payload(decode=True))
content = content.decode()
a = {
'filename': email_nonascii_to_uft8(filename),
'content': content,
'content-type': part.get_content_type(),
}
attachments.append(a)
else:
part_item = {}
content = part.get_payload(decode=True)
content_type = part.get_content_type()
try:
charset = part.get_param('charset', None)
if charset:
content = to_utf8(content, charset)
elif type(content) == bytes:
content = content.decode('utf8')
except:
self.logger.exception()
# RFC 3676: remove automatic word-wrapping
content = content.replace(' \r\n', ' ')
part_item['content'] = content
part_item['content-type'] = content_type
parts.append(part_item)
if parts:
msg['parts'] = parts
if attachments:
msg['attachments'] = attachments
return msg
def delete_message(self, num):
self.imap.select('Inbox')
self.imap.store(str(num), '+FLAGS', r'\Deleted')
self.imap.expunge()
def delete_all(self):
self.imap.select('Inbox')
_, data = self.imap.search(None, 'ALL')
for num in data[0].split():
self.imap.store(num, '+FLAGS', r'\Deleted')
self.imap.expunge()
def print_msgs(self):
self.imap.select('Inbox')
_, data = self.imap.search(None, 'ALL')
for num in reversed(data[0].split()):
status, data = self.imap.fetch(num, '(RFC822)')
self.logger.debug('Message %s\n%s\n' % (num, data[0][1]))
def parse_date(v):
if v is None:
return datetime.datetime.now()
tt = email.utils.parsedate_tz(v)
if tt is None:
return datetime.datetime.now()
timestamp = email.utils.mktime_tz(tt)
date = datetime.datetime.fromtimestamp(timestamp)
return date
def to_utf8(string, charset):
return string.decode(charset).encode('UTF-8').decode('UTF-8')
def email_nonascii_to_uft8(string):
# RFC 1342 is a recommendation that provides a way to represent non ASCII
# characters inside e-mail in a way that won’t confuse e-mail servers
subject = ''
for v, charset in email.header.decode_header(string):
if charset is None:
if type(v) is bytes:
v = v.decode()
subject = subject + v
else:
subject = subject + to_utf8(v, charset)
return subject

94
stacosys/core/mailer.py
View file

@ -1,43 +1,85 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging
import json import json
import logging
import smtplib
from email.mime.text import MIMEText
import requests import requests
from conf import config from conf import config
from core import imap
from model.email import Email
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def _open_mailbox():
return imap.Mailbox(
config.get(config.IMAP_HOST),
config.get_int(config.IMAP_PORT),
config.get_bool(config.IMAP_SSL),
config.get(config.IMAP_LOGIN),
config.get(config.IMAP_PASSWORD),
)
def _to_dto(msg):
content = 'no plain-text part found in email'
for part in msg['parts']:
if part['content-type'] == 'text/plain':
content = part['content']
break
return Email(
id=msg['index'],
encoding=msg['encoding'],
date=msg['datetime'],
from_addr=msg['from'],
to_addr=msg['to'],
subject=msg['subject'],
content=content,
)
def fetch(): def fetch():
mails = [] msgs = []
r = requests.get(config.get(config.MAILER_URL) + "/mbox") try:
if r.status_code == 200: with _open_mailbox() as mbox:
payload = r.json() count = mbox.get_count()
if payload["count"] > 0: for num in range(count):
mails = payload["emails"] msg = _to_dto(mbox.fetch_message(num + 1))
return mails msgs.append(msg)
except:
logger.exception('fetch mail exception')
def get(id): return msgs
payload = None
r = requests.get(config.get(config.MAILER_URL) + "/mbox/" + str(id))
if r.status_code == 200:
payload = r.json()
return payload
def send(to_email, subject, message): def send(to_email, subject, message):
headers = {"Content-Type": "application/json; charset=utf-8"}
msg = {"to": to_email, "subject": subject, "content": message} # Create the container (outer) email message.
r = requests.post( msg = MIMEText(message)
config.get(config.MAILER_URL) + "/mbox", data=json.dumps(msg), headers=headers msg['Subject'] = subject
) msg['To'] = to_email
if r.status_code in (200, 201): msg['From'] = config.get(config.SMTP_LOGIN)
logger.debug("Email for %s posted" % to_email)
else: success = True
logger.warn("Cannot post email for %s" % to_email) try:
s = smtplib.SMTP(config.get(config.SMTP_HOST), config.getInt(config.SMTP_PORT))
if config.get_bool(config.SMTP_STARTTLS):
s.starttls()
s.login(config.get(config.SMTP_LOGIN), config.get(config.SMTP_PASSWORD))
s.send_message(msg)
s.quit()
except:
logger.exception('send mail exception')
success = False
return success
def delete(id): def delete(id):
requests.delete(config.get(config.MAILER_URL) + "/mbox/" + str(id)) try:
with _open_mailbox() as mbox:
mbox.delete_message(id)
except:
logger.exception('delete mail exception')

24
stacosys/core/rss.py
View file

@ -2,15 +2,18 @@
# -*- coding: UTF-8 -*- # -*- coding: UTF-8 -*-
from datetime import datetime from datetime import datetime
import markdown import markdown
import PyRSS2Gen import PyRSS2Gen
from model.site import Site
from model.comment import Comment
from core.templater import get_template
from conf import config from conf import config
from core.templater import get_template
from model.comment import Comment
from model.site import Site
def generate_all(): def generate_all():
for site in Site.select(): for site in Site.select():
generate_site(site.token) generate_site(site.token)
@ -18,7 +21,7 @@ def generate_all():
def generate_site(token): def generate_site(token):
site = Site.select().where(Site.token == token).get() site = Site.select().where(Site.token == token).get()
rss_title = get_template("rss_title_message").render(site=site.name) rss_title = get_template('rss_title_message').render(site=site.name)
md = markdown.Markdown() md = markdown.Markdown()
items = [] items = []
@ -29,24 +32,23 @@ def generate_site(token):
.order_by(-Comment.published) .order_by(-Comment.published)
.limit(10) .limit(10)
): ):
item_link = "%s://%s%s" % (config.get(config.RSS_PROTO), site.url, row.url) item_link = '%s://%s%s' % (config.get(config.RSS_PROTO), site.url, row.url)
items.append( items.append(
PyRSS2Gen.RSSItem( PyRSS2Gen.RSSItem(
title="%s - %s://%s%s" title='%s - %s://%s%s'
% (config.get(config.RSS_PROTO), row.author_name, site.url, row.url), % (config.get(config.RSS_PROTO), row.author_name, site.url, row.url),
link=item_link, link=item_link,
description=md.convert(row.content), description=md.convert(row.content),
guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)), guid=PyRSS2Gen.Guid('%s/%d' % (item_link, row.id)),
pubDate=row.published, pubDate=row.published,
) )
) )
rss = PyRSS2Gen.RSS2( rss = PyRSS2Gen.RSS2(
title=rss_title, title=rss_title,
link="%s://%s" % (config.get(config.RSS_PROTO), site.url), link='%s://%s' % (config.get(config.RSS_PROTO), site.url),
description="Commentaires du site '%s'" % site.name, description='Commentaires du site "%s"' % site.name,
lastBuildDate=datetime.now(), lastBuildDate=datetime.now(),
items=items, items=items,
) )
rss.write_xml(open(config.get(config.RSS_FILE), "w"), encoding="utf-8") rss.write_xml(open(config.get(config.RSS_FILE), 'w'), encoding='utf-8')

9
stacosys/core/templater.py
View file

@ -2,14 +2,15 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os import os
from jinja2 import Environment
from jinja2 import FileSystemLoader from jinja2 import Environment, FileSystemLoader
from conf import config from conf import config
current_path = os.path.dirname(__file__) current_path = os.path.dirname(__file__)
template_path = os.path.abspath(os.path.join(current_path, "../templates")) template_path = os.path.abspath(os.path.join(current_path, '../templates'))
env = Environment(loader=FileSystemLoader(template_path)) env = Environment(loader=FileSystemLoader(template_path))
def get_template(name): def get_template(name):
return env.get_template(config.get(config.LANG) + "/" + name + ".tpl") return env.get_template(config.get(config.LANG) + '/' + name + '.tpl')

16
stacosys/helper/hashing.py
View file

@ -1,16 +0,0 @@
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import hashlib
from conf import config
def salt(value):
string = "%s%s" % (value, config.get(config.SECURITY_SALT))
dk = hashlib.sha256(string.encode())
return dk.hexdigest()
def md5(value):
dk = hashlib.md5(value.encode())
return dk.hexdigest()

49
stacosys/interface/api.py
View file

@ -2,29 +2,31 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging import logging
from flask import request, jsonify, abort
from model.site import Site from flask import abort, jsonify, request
from model.comment import Comment
from conf import config from conf import config
from model.comment import Comment
from model.site import Site
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
app = config.flaskapp() app = config.flaskapp()
@app.route("/ping", methods=["GET"]) @app.route('/ping', methods=['GET'])
def ping(): def ping():
return "OK" return 'OK'
@app.route("/comments", methods=["GET"]) @app.route('/comments', methods=['GET'])
def query_comments(): def query_comments():
comments = [] comments = []
try: try:
token = request.args.get("token", "") token = request.args.get('token', '')
url = request.args.get("url", "") url = request.args.get('url', '')
logger.info("retrieve comments for token %s, url %s" % (token, url)) logger.info('retrieve comments for url %s' % (url))
for comment in ( for comment in (
Comment.select(Comment) Comment.select(Comment)
.join(Site) .join(Site)
@ -35,29 +37,30 @@ def query_comments():
) )
.order_by(+Comment.published) .order_by(+Comment.published)
): ):
d = {} d = {
d["author"] = comment.author_name 'author': comment.author_name,
d["content"] = comment.content 'content': comment.content,
'avatar': comment.author_gravatar,
'date': comment.published.strftime('%Y-%m-%d %H:%M:%S')
}
if comment.author_site: if comment.author_site:
d["site"] = comment.author_site d['site'] = comment.author_site
d["avatar"] = comment.author_gravatar
d["date"] = comment.published.strftime("%Y-%m-%d %H:%M:%S")
logger.debug(d) logger.debug(d)
comments.append(d) comments.append(d)
r = jsonify({"data": comments}) r = jsonify({'data': comments})
r.status_code = 200 r.status_code = 200
except: except:
logger.warn("bad request") logger.warn('bad request')
r = jsonify({"data": []}) r = jsonify({'data': []})
r.status_code = 400 r.status_code = 400
return r return r
@app.route("/comments/count", methods=["GET"]) @app.route('/comments/count', methods=['GET'])
def get_comments_count(): def get_comments_count():
try: try:
token = request.args.get("token", "") token = request.args.get('token', '')
url = request.args.get("url", "") url = request.args.get('url', '')
count = ( count = (
Comment.select(Comment) Comment.select(Comment)
.join(Site) .join(Site)
@ -68,9 +71,9 @@ def get_comments_count():
) )
.count() .count()
) )
r = jsonify({"count": count}) r = jsonify({'count': count})
r.status_code = 200 r.status_code = 200
except: except:
r = jsonify({"count": 0}) r = jsonify({'count': 0})
r.status_code = 200 r.status_code = 200
return r return r

47
stacosys/interface/form.py
View file

@ -3,52 +3,53 @@
import logging import logging
from datetime import datetime from datetime import datetime
from flask import request, abort, redirect
from model.site import Site from flask import abort, redirect, request
from model.comment import Comment
from conf import config from conf import config
from helper.hashing import md5 from model.comment import Comment
from model.site import Site
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
app = config.flaskapp() app = config.flaskapp()
@app.route("/newcomment", methods=["POST"]) @app.route('/newcomment', methods=['POST'])
def new_form_comment(): def new_form_comment():
try: try:
data = request.form data = request.form
logger.info("form data " + str(data)) logger.info('form data ' + str(data))
# validate token: retrieve site entity # validate token: retrieve site entity
token = data.get("token", "") token = data.get('token', '')
site = Site.select().where(Site.token == token).get() site = Site.select().where(Site.token == token).get()
if site is None: if site is None:
logger.warn("Unknown site %s" % token) logger.warn('Unknown site %s' % token)
abort(400) abort(400)
# honeypot for spammers # honeypot for spammers
captcha = data.get("remarque", "") captcha = data.get('remarque', '')
if captcha: if captcha:
logger.warn("discard spam: data %s" % data) logger.warn('discard spam: data %s' % data)
abort(400) abort(400)
url = data.get("url", "") url = data.get('url', '')
author_name = data.get("author", "").strip() author_name = data.get('author', '').strip()
author_gravatar = data.get("email", "").strip() author_gravatar = data.get('email', '').strip()
author_site = data.get("site", "").lower().strip() author_site = data.get('site', '').lower().strip()
if author_site and author_site[:4] != "http": if author_site and author_site[:4] != 'http':
author_site = "http://" + author_site author_site = 'http://' + author_site
message = data.get("message", "") message = data.get('message', '')
# anti-spam again # anti-spam again
if not url or not author_name or not message: if not url or not author_name or not message:
logger.warn("empty field: data %s" % data) logger.warn('empty field: data %s' % data)
abort(400) abort(400)
check_form_data(data) check_form_data(data)
# add a row to Comment table # add a row to Comment table
created = datetime.now().strftime("%Y-%m-%d %H:%M:%S") created = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
comment = Comment( comment = Comment(
site=site, site=site,
url=url, url=url,
@ -63,18 +64,18 @@ def new_form_comment():
comment.save() comment.save()
except: except:
logger.exception("new comment failure") logger.exception('new comment failure')
abort(400) abort(400)
return redirect("/redirect/", code=302) return redirect('/redirect/', code=302)
def check_form_data(data): def check_form_data(data):
fields = ["url", "message", "site", "remarque", "author", "token", "email"] fields = ['url', 'message', 'site', 'remarque', 'author', 'token', 'email']
d = data.to_dict() d = data.to_dict()
for field in fields: for field in fields:
if field in d: if field in d:
del d[field] del d[field]
if d: if d:
logger.warn("additional field: data %s" % data) logger.warn('additional field: data %s' % data)
abort(400) abort(400)

10
stacosys/model/comment.py
View file

@ -17,18 +17,18 @@ class Comment(Model):
notified = DateTimeField(null=True, default=None) notified = DateTimeField(null=True, default=None)
published = DateTimeField(null=True, default=None) published = DateTimeField(null=True, default=None)
author_name = CharField() author_name = CharField()
author_site = CharField(default="") author_site = CharField(default='')
author_gravatar = CharField(default="") author_gravatar = CharField(default='')
content = TextField() content = TextField()
site = ForeignKeyField(Site, related_name="site") site = ForeignKeyField(Site, related_name='site')
class Meta: class Meta:
database = get_db() database = get_db()
def notify_site_admin(self): def notify_site_admin(self):
self.notified = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.notified = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.save() self.save()
def publish(self): def publish(self):
self.published = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.published = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.save() self.save()

14
stacosys/model/email.py Normal file
View file

@ -0,0 +1,14 @@
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from typing import NamedTuple
from datetime import datetime
class Email(NamedTuple):
id: int
encoding: str
date: datetime
from_addr: str
to_addr: str
subject: str
content: str

45
stacosys/run.py
View file

@ -2,12 +2,15 @@
# -*- coding: UTF-8 -*- # -*- coding: UTF-8 -*-
import argparse import argparse
import os
import logging import logging
import os
from flask import Flask from flask import Flask
from flask_apscheduler import APScheduler from flask_apscheduler import APScheduler
from conf import config from conf import config
# configure logging # configure logging
def configure_logging(level): def configure_logging(level):
root_logger = logging.getLogger() root_logger = logging.getLogger()
@ -15,7 +18,7 @@ def configure_logging(level):
ch = logging.StreamHandler() ch = logging.StreamHandler()
ch.setLevel(level) ch.setLevel(level)
# create formatter # create formatter
formatter = logging.Formatter("[%(asctime)s] %(name)s %(levelname)s %(message)s") formatter = logging.Formatter('[%(asctime)s] %(name)s %(levelname)s %(message)s')
# add formatter to ch # add formatter to ch
ch.setFormatter(formatter) ch.setFormatter(formatter)
# add ch to logger # add ch to logger
@ -26,21 +29,21 @@ class JobConfig(object):
JOBS = [] JOBS = []
SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 20}} SCHEDULER_EXECUTORS = {'default': {'type': 'threadpool', 'max_workers': 4}}
def __init__(self, mail_polling_seconds, new_comment_polling_seconds): def __init__(self, imap_polling_seconds, new_comment_polling_seconds):
self.JOBS = [ self.JOBS = [
{ {
"id": "fetch_mail", 'id': 'fetch_mail',
"func": "core.cron:fetch_mail_answers", 'func': 'core.cron:fetch_mail_answers',
"trigger": "interval", 'trigger': 'interval',
"seconds": mail_polling_seconds, 'seconds': imap_polling_seconds,
}, },
{ {
"id": "submit_new_comment", 'id': 'submit_new_comment',
"func": "core.cron:submit_new_comment", 'func': 'core.cron:submit_new_comment',
"trigger": "interval", 'trigger': 'interval',
"seconds": new_comment_polling_seconds, 'seconds': new_comment_polling_seconds,
}, },
] ]
@ -53,8 +56,8 @@ def stacosys_server(config_pathname):
# configure logging # configure logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
configure_logging(logging.INFO) configure_logging(logging.INFO)
logging.getLogger("werkzeug").level = logging.WARNING logging.getLogger('werkzeug').level = logging.WARNING
logging.getLogger("apscheduler.executors").level = logging.WARNING logging.getLogger('apscheduler.executors').level = logging.WARNING
# initialize database # initialize database
from core import database from core import database
@ -64,14 +67,14 @@ def stacosys_server(config_pathname):
# cron email fetcher # cron email fetcher
app.config.from_object( app.config.from_object(
JobConfig( JobConfig(
config.getInt(config.MAIL_POLLING), config.getInt(config.COMMENT_POLLING) config.get_int(config.IMAP_POLLING), config.get_int(config.COMMENT_POLLING)
) )
) )
scheduler = APScheduler() scheduler = APScheduler()
scheduler.init_app(app) scheduler.init_app(app)
scheduler.start() scheduler.start()
logger.info("Start Stacosys application") logger.info('Start Stacosys application')
# generate RSS for all sites # generate RSS for all sites
from core import rss from core import rss
@ -80,10 +83,12 @@ def stacosys_server(config_pathname):
# start Flask # start Flask
from interface import api from interface import api
logger.info('Load interface %s' % api)
from interface import form from interface import form
logger.debug("Load interface %s" % api) logger.info('Load interface %s' % form)
logger.debug("Load interface %s" % form)
app.run( app.run(
host=config.get(config.HTTP_HOST), host=config.get(config.HTTP_HOST),
@ -93,8 +98,8 @@ def stacosys_server(config_pathname):
) )
if __name__ == "__main__": if __name__ == '__main__':
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("config", help="config path name") parser.add_argument('config', help='config path name')
args = parser.parse_args() args = parser.parse_args()
stacosys_server(args.config) stacosys_server(args.config)