yannic/stacosys
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve encapsulation

Browse source
This commit is contained in:
Yax 2021-01-03 18:24:16 +01:00
parent 6c855e7ead
commit adc6451116
7 changed files with 240 additions and 196 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
run.py
View file

@ -1,19 +1,21 @@
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import sys
import os
import argparse
import logging
import os
import sys
from flask import Flask
from flask_apscheduler import APScheduler
import stacosys.conf.config as config
from stacosys.core import database
from stacosys.core import rss
#from stacosys.interface import api
#from stacosys.interface import form
from stacosys.core.rss import Rss
from stacosys.core.mailer import Mailer
from stacosys.interface import app
from stacosys.interface import api
from stacosys.interface import form
from stacosys.interface import scheduler
# configure logging
def configure_logging(level):
@ -29,33 +31,8 @@ def configure_logging(level):
root_logger.addHandler(ch)
class JobConfig(object):
JOBS = []
SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 4}}
def __init__(self, imap_polling_seconds, new_comment_polling_seconds):
self.JOBS = [
{
"id": "fetch_mail",
"func": "stacosys.core.cron:fetch_mail_answers",
"trigger": "interval",
"seconds": imap_polling_seconds,
},
{
"id": "submit_new_comment",
"func": "stacosys.core.cron:submit_new_comment",
"trigger": "interval",
"seconds": new_comment_polling_seconds,
},
]
def stacosys_server(config_pathname):
app = Flask(__name__)
conf = config.Config.load(config_pathname)
# configure logging
@ -68,26 +45,38 @@ def stacosys_server(config_pathname):
db = database.Database()
db.setup(conf.get(config.DB_URL))
# cron email fetcher
app.config.from_object(
JobConfig(
conf.get_int(config.IMAP_POLLING), conf.get_int(config.COMMENT_POLLING)
)
)
scheduler = APScheduler()
scheduler.init_app(app)
scheduler.start()
logger.info("Start Stacosys application")
# generate RSS for all sites
rss_manager = rss.Rss(conf.get(config.LANG), conf.get(config.RSS_FILE), conf.get(config.RSS_PROTO))
rss_manager.generate_all()
rss = Rss(
conf.get(config.LANG), conf.get(config.RSS_FILE), conf.get(config.RSS_PROTO)
)
rss.generate_all()
# configure mailer
mailer = Mailer(
conf.get(config.IMAP_HOST),
conf.get_int(config.IMAP_PORT),
conf.get_bool(config.IMAP_SSL),
conf.get(config.IMAP_LOGIN),
conf.get(config.IMAP_PASSWORD),
conf.get(config.SMTP_HOST),
conf.get_int(config.SMTP_PORT),
conf.get_bool(config.SMTP_STARTTLS),
conf.get(config.SMTP_LOGIN),
conf.get(config.SMTP_PASSWORD),
)
# configure scheduler
scheduler.configure(
conf.get_int(config.IMAP_POLLING),
conf.get_int(config.COMMENT_POLLING),
conf.get(config.LANG),
mailer,
rss,
)
# start Flask
#logger.info("Load interface %s" % api)
#logger.info("Load interface %s" % form)
app.run(
host=conf.get(config.HTTP_HOST),
port=conf.get(config.HTTP_PORT),

117
stacosys/core/cron.py
View file

@ -6,7 +6,7 @@ import re
import time
from datetime import datetime
from stacosys.core import mailer, rss
from stacosys.core import rss
from stacosys.core.templater import get_template
from stacosys.model.comment import Comment, Site
from stacosys.model.email import Email
@ -14,59 +14,18 @@ from stacosys.model.email import Email
logger = logging.getLogger(__name__)
def cron(func):
def wrapper():
logger.debug('execute CRON ' + func.__name__)
func()
return wrapper
@cron
def fetch_mail_answers():
def fetch_mail_answers(lang, mailer, rss):
for msg in mailer.fetch():
if re.search(r'.*STACOSYS.*\[(\d+)\:(\w+)\]', msg.subject, re.DOTALL):
if _reply_comment_email(msg):
if re.search(r".*STACOSYS.*\[(\d+)\:(\w+)\]", msg.subject, re.DOTALL):
if _reply_comment_email(lang, mailer, rss, msg):
mailer.delete(msg.id)
@cron
def submit_new_comment():
def _reply_comment_email(lang, mailer, rss, email: Email):
for comment in Comment.select().where(Comment.notified.is_null()):
comment_list = (
'author: %s' % comment.author_name,
'site: %s' % comment.author_site,
'date: %s' % comment.created,
'url: %s' % comment.url,
'',
'%s' % comment.content,
'',
)
comment_text = '\n'.join(comment_list)
email_body = get_template('new_comment').render(
url=comment.url, comment=comment_text
)
# send email
site = Site.get(Site.id == comment.site)
subject = 'STACOSYS %s: [%d:%s]' % (site.name, comment.id, site.token)
if mailer.send(site.admin_email, subject, email_body):
logger.debug('new comment processed ')
# notify site admin and save notification datetime
comment.notify_site_admin()
else:
logger.warn('rescheduled. send mail failure ' + subject)
def _reply_comment_email(email : Email):
m = re.search(r'\[(\d+)\:(\w+)\]', email.subject)
m = re.search(r"\[(\d+)\:(\w+)\]", email.subject)
if not m:
logger.warn('ignore corrupted email. No token %s' % email.subject)
logger.warn("ignore corrupted email. No token %s" % email.subject)
return
comment_id = int(m.group(1))
token = m.group(2)
@ -75,39 +34,75 @@ def _reply_comment_email(email : Email):
try:
comment = Comment.select().where(Comment.id == comment_id).get()
except:
logger.warn('unknown comment %d' % comment_id)
logger.warn("unknown comment %d" % comment_id)
return True
if comment.published:
logger.warn('ignore already published email. token %d' % comment_id)
logger.warn("ignore already published email. token %d" % comment_id)
return
if comment.site.token != token:
logger.warn('ignore corrupted email. Unknown token %d' % comment_id)
logger.warn("ignore corrupted email. Unknown token %d" % comment_id)
return
if not email.plain_text_content:
logger.warn('ignore empty email')
logger.warn("ignore empty email")
return
# safe logic: no answer or unknown answer is a go for publishing
if email.plain_text_content[:2].upper() in ('NO'):
logger.info('discard comment: %d' % comment_id)
if email.plain_text_content[:2].upper() in ("NO"):
logger.info("discard comment: %d" % comment_id)
comment.delete_instance()
new_email_body = get_template('drop_comment').render(original=email.plain_text_content)
if not mailer.send(email.from_addr, 'Re: ' + email.subject, new_email_body):
logger.warn('minor failure. cannot send rejection mail ' + email.subject)
new_email_body = get_template(lang, "drop_comment").render(
original=email.plain_text_content
)
if not mailer.send(email.from_addr, "Re: " + email.subject, new_email_body):
logger.warn("minor failure. cannot send rejection mail " + email.subject)
else:
# save publishing datetime
comment.publish()
logger.info('commit comment: %d' % comment_id)
logger.info("commit comment: %d" % comment_id)
# rebuild RSS
rss.generate_site(token)
# send approval confirmation email to admin
new_email_body = get_template('approve_comment').render(original=email.plain_text_content)
if not mailer.send(email.from_addr, 'Re: ' + email.subject, new_email_body):
logger.warn('minor failure. cannot send approval email ' + email.subject)
new_email_body = get_template(lang, "approve_comment").render(
original=email.plain_text_content
)
if not mailer.send(email.from_addr, "Re: " + email.subject, new_email_body):
logger.warn("minor failure. cannot send approval email " + email.subject)
return True
def submit_new_comment(lang, mailer):
for comment in Comment.select().where(Comment.notified.is_null()):
comment_list = (
"author: %s" % comment.author_name,
"site: %s" % comment.author_site,
"date: %s" % comment.created,
"url: %s" % comment.url,
"",
"%s" % comment.content,
"",
)
comment_text = "\n".join(comment_list)
# TODO use constants for template names
email_body = get_template(lang, "new_comment").render(
url=comment.url, comment=comment_text
)
# send email
site = Site.get(Site.id == comment.site)
subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token)
if mailer.send(site.admin_email, subject, email_body):
logger.debug("new comment processed ")
# notify site admin and save notification datetime
comment.notify_site_admin()
else:
logger.warn("rescheduled. send mail failure " + subject)

110
stacosys/core/mailer.py
View file

@ -15,53 +15,75 @@ from stacosys.model.email import Email
logger = logging.getLogger(__name__)
def _open_mailbox():
return imap.Mailbox(
config.get(config.IMAP_HOST),
config.get_int(config.IMAP_PORT),
config.get_bool(config.IMAP_SSL),
config.get(config.IMAP_LOGIN),
config.get(config.IMAP_PASSWORD),
)
class Mailer:
def __init__(
self,
imap_host,
imap_port,
imap_ssl,
imap_login,
imap_password,
smtp_host,
smtp_port,
smtp_starttls,
smtp_login,
smtp_password,
):
self._imap_host = imap_host
self._imap_port = imap_port
self._imap_ssl = imap_ssl
self._imap_login = imap_login
self._imap_password = imap_password
self._smtp_host = smtp_host
self._smtp_port = smtp_port
self._smtp_starttls = smtp_starttls
self._smtp_login = smtp_login
self._smtp_password = smtp_password
def _open_mailbox(self):
return imap.Mailbox(
self._imap_host,
self._imap_port,
self._imap_ssl,
self._imap_login,
self._imap_password,
)
def fetch():
msgs = []
try:
with _open_mailbox() as mbox:
count = mbox.get_count()
for num in range(count):
msgs.append(mbox.fetch_message(num + 1))
except:
logger.exception("fetch mail exception")
return msgs
def fetch(self):
msgs = []
try:
with self._open_mailbox() as mbox:
count = mbox.get_count()
for num in range(count):
msgs.append(mbox.fetch_message(num + 1))
except:
logger.exception("fetch mail exception")
return msgs
def send(self, to_email, subject, message):
def send(to_email, subject, message):
# Create the container (outer) email message.
msg = MIMEText(message)
msg["Subject"] = subject
msg["To"] = to_email
msg["From"] = self._smtp_login
# Create the container (outer) email message.
msg = MIMEText(message)
msg["Subject"] = subject
msg["To"] = to_email
msg["From"] = config.get(config.SMTP_LOGIN)
success = True
try:
s = smtplib.SMTP(self._smtp_host, self._smtp_port)
if self._smtp_starttls:
s.starttls()
s.login(self._smtp_login, self._smtp_password)
s.send_message(msg)
s.quit()
except:
logger.exception("send mail exception")
success = False
return success
success = True
try:
s = smtplib.SMTP(config.get(config.SMTP_HOST), config.get_int(config.SMTP_PORT))
if config.get_bool(config.SMTP_STARTTLS):
s.starttls()
s.login(config.get(config.SMTP_LOGIN), config.get(config.SMTP_PASSWORD))
s.send_message(msg)
s.quit()
except:
logger.exception("send mail exception")
success = False
return success
def delete(id):
try:
with _open_mailbox() as mbox:
mbox.delete_message(id)
except:
logger.exception("delete mail exception")
def delete(self, id):
try:
with self._open_mailbox() as mbox:
mbox.delete_message(id)
except:
logger.exception("delete mail exception")

5
stacosys/interface/__init__.py
View file

@ -0,0 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from flask import Flask
app = Flask(__name__)

42
stacosys/interface/api.py
View file

@ -2,31 +2,29 @@
# -*- coding: utf-8 -*-
import logging
from flask import abort, jsonify, request
from stacosys.conf import config
from stacosys.interface import app
from stacosys.model.comment import Comment
from stacosys.model.site import Site
logger = logging.getLogger(__name__)
app = config.flaskapp()
@app.route('/ping', methods=['GET'])
@app.route("/ping", methods=["GET"])
def ping():
return 'OK'
return "OK"
@app.route('/comments', methods=['GET'])
@app.route("/comments", methods=["GET"])
def query_comments():
comments = []
try:
token = request.args.get('token', '')
url = request.args.get('url', '')
token = request.args.get("token", "")
url = request.args.get("url", "")
logger.info('retrieve comments for url %s' % (url))
logger.info("retrieve comments for url %s" % (url))
for comment in (
Comment.select(Comment)
.join(Site)
@ -38,29 +36,29 @@ def query_comments():
.order_by(+Comment.published)
):
d = {
'author': comment.author_name,
'content': comment.content,
'avatar': comment.author_gravatar,
'date': comment.published.strftime('%Y-%m-%d %H:%M:%S')
"author": comment.author_name,
"content": comment.content,
"avatar": comment.author_gravatar,
"date": comment.published.strftime("%Y-%m-%d %H:%M:%S"),
}
if comment.author_site:
d['site'] = comment.author_site
d["site"] = comment.author_site
logger.debug(d)
comments.append(d)
r = jsonify({'data': comments})
r = jsonify({"data": comments})
r.status_code = 200
except:
logger.warn('bad request')
r = jsonify({'data': []})
logger.warn("bad request")
r = jsonify({"data": []})
r.status_code = 400
return r
@app.route('/comments/count', methods=['GET'])
@app.route("/comments/count", methods=["GET"])
def get_comments_count():
try:
token = request.args.get('token', '')
url = request.args.get('url', '')
token = request.args.get("token", "")
url = request.args.get("url", "")
count = (
Comment.select(Comment)
.join(Site)
@ -71,9 +69,9 @@ def get_comments_count():
)
.count()
)
r = jsonify({'count': count})
r = jsonify({"count": count})
r.status_code = 200
except:
r = jsonify({'count': 0})
r = jsonify({"count": 0})
r.status_code = 200
return r

42
stacosys/interface/form.py
View file

@ -3,53 +3,51 @@
import logging
from datetime import datetime
from flask import abort, redirect, request
from stacosys.conf import config
from stacosys.interface import app
from stacosys.model.comment import Comment
from stacosys.model.site import Site
logger = logging.getLogger(__name__)
app = config.flaskapp()
@app.route('/newcomment', methods=['POST'])
@app.route("/newcomment", methods=["POST"])
def new_form_comment():
try:
data = request.form
logger.info('form data ' + str(data))
logger.info("form data " + str(data))
# validate token: retrieve site entity
token = data.get('token', '')
token = data.get("token", "")
site = Site.select().where(Site.token == token).get()
if site is None:
logger.warn('Unknown site %s' % token)
logger.warn("Unknown site %s" % token)
abort(400)
# honeypot for spammers
captcha = data.get('remarque', '')
captcha = data.get("remarque", "")
if captcha:
logger.warn('discard spam: data %s' % data)
logger.warn("discard spam: data %s" % data)
abort(400)
url = data.get('url', '')
author_name = data.get('author', '').strip()
author_gravatar = data.get('email', '').strip()
author_site = data.get('site', '').lower().strip()
if author_site and author_site[:4] != 'http':
author_site = 'http://' + author_site
message = data.get('message', '')
url = data.get("url", "")
author_name = data.get("author", "").strip()
author_gravatar = data.get("email", "").strip()
author_site = data.get("site", "").lower().strip()
if author_site and author_site[:4] != "http":
author_site = "http://" + author_site
message = data.get("message", "")
# anti-spam again
if not url or not author_name or not message:
logger.warn('empty field: data %s' % data)
logger.warn("empty field: data %s" % data)
abort(400)
check_form_data(data)
# add a row to Comment table
created = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
created = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment = Comment(
site=site,
url=url,
@ -64,18 +62,18 @@ def new_form_comment():
comment.save()
except:
logger.exception('new comment failure')
logger.exception("new comment failure")
abort(400)
return redirect('/redirect/', code=302)
return redirect("/redirect/", code=302)
def check_form_data(data):
fields = ['url', 'message', 'site', 'remarque', 'author', 'token', 'email']
fields = ["url", "message", "site", "remarque", "author", "token", "email"]
d = data.to_dict()
for field in fields:
if field in d:
del d[field]
if d:
logger.warn('additional field: data %s' % data)
logger.warn("additional field: data %s" % data)
abort(400)

37
stacosys/interface/scheduler.py Normal file
View file

@ -0,0 +1,37 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from flask_apscheduler import APScheduler
from stacosys.interface import app
class JobConfig(object):
JOBS = []
SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 4}}
def __init__(self, imap_polling_seconds, new_comment_polling_seconds, lang, mailer, rss):
self.JOBS = [
{
"id": "fetch_mail",
"func": "stacosys.core.cron:fetch_mail_answers",
"args": [lang, mailer, rss],
"trigger": "interval",
"seconds": imap_polling_seconds,
},
{
"id": "submit_new_comment",
"func": "stacosys.core.cron:submit_new_comment",
"args": [lang, mailer],
"trigger": "interval",
"seconds": new_comment_polling_seconds,
},
]
def configure(imap_polling, comment_polling, lang, mailer, rss):
app.config.from_object(JobConfig(imap_polling, comment_polling, lang, mailer, rss))
scheduler = APScheduler()
scheduler.init_app(app)
scheduler.start()