yannic/stacosys
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WIP

Browse source
This commit is contained in:
Yax 2018-09-15 15:02:06 +02:00
parent 0a2cdbbe8f
commit 9146588902
11 changed files with 185 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
app/__init__.py
View file

10
app/conf/config.py
View file

@ -7,6 +7,7 @@ import profig
FLASK_APP = "flask.app"
DB_URL = "main.db_url"
LANG = "main.lang"
HTTP_HOST = "http.host"
HTTP_PORT = "http.port"
@ -14,8 +15,13 @@ HTTP_PORT = "http.port"
SECURITY_SALT = "security.salt"
SECURITY_SECRET = "security.secret"
MAIL_POLLING = "polling.newmail"
COMMENT_POLLING = "polling.newcomment"
RSS_PROTO = "rss.proto"
RSS_FILE = "rss.file"
MAIL_POLLING = "mail.fetch_polling"
COMMENT_POLLING = "main.newcomment_polling"
MAILER_URL = "mail.mailer_url"
# variable
params = dict()

105
app/core/cron.py
View file

@ -2,30 +2,48 @@
# -*- coding: utf-8 -*-
import logging
from datetime import datetime
import time
import re
from core import mailer
from core import templater
from core.templater import get_template
from core import rss
from model.comment import Comment
from model.comment import Site
logger = logging.getLogger(__name__)
client_ips = {}
def cron(func):
def wrapper():
logger.debug("execute fun " + func)
func()
return wrapper
@cron
def fetch_mail_answers():
logger.info("DEBUT POP MAIL")
time.sleep(80)
logger.info("FIN POP MAIL")
msg = {}
if msg["request"] == "new_mail":
reply_comment_email(msg["data"])
mailer.delete(msg["data"])
# data = request.get_json()
# logger.debug(data)
# processor.enqueue({'request': 'new_mail', 'data': data})
@cron
def submit_new_comment():
for comment in Comment.select().where(Comment.notified.is_null()):
comment_list = (
"author: %s" % comment.author_name,
"site: %s" % comment.author_site,
@ -36,10 +54,83 @@ def submit_new_comment():
"",
)
comment_text = "\n".join(comment_list)
email_body = templater.get_template("new_comment").render(url=comment.url, comment=comment_text)
email_body = get_template("new_comment").render(
url=comment.url, comment=comment_text
)
site = Site.select().where(Site.id == Comment.site).get()
# send email
subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token)
mailer.send_mail(site.admin_email, subject, email_body)
mailer.send(site.admin_email, subject, email_body)
logger.debug("new comment processed ")
def reply_comment_email(data):
from_email = data["from"]
subject = data["subject"]
message = ""
for part in data["parts"]:
if part["content-type"] == "text/plain":
message = part["content"]
break
m = re.search(r"\[(\d+)\:(\w+)\]", subject)
if not m:
logger.warn("ignore corrupted email. No token %s" % subject)
return
comment_id = int(m.group(1))
token = m.group(2)
# retrieve site and comment rows
try:
comment = Comment.select().where(Comment.id == comment_id).get()
except:
logger.warn("unknown comment %d" % comment_id)
return
if comment.published:
logger.warn("ignore already published email. token %d" % comment_id)
return
if comment.site.token != token:
logger.warn("ignore corrupted email. Unknown token %d" % comment_id)
return
if not message:
logger.warn("ignore empty email")
return
# safe logic: no answer or unknown answer is a go for publishing
if message[:2].upper() in ("NO", "SP"):
# put a log to help fail2ban
if message[:2].upper() == "SP": # SPAM
if comment_id in client_ips:
logger.info(
"SPAM comment from %s: %d" % (client_ips[comment_id], comment_id)
)
else:
logger.info("cannot identify SPAM source: %d" % comment_id)
# forget client IP
if comment_id in client_ips:
del client_ips[comment_id]
logger.info("discard comment: %d" % comment_id)
comment.delete_instance()
email_body = get_template("drop_comment").render(original=message)
mailer.send(from_email, "Re: " + subject, email_body)
else:
# update Comment row
comment.published = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment.save()
logger.info("commit comment: %d" % comment_id)
# rebuild RSS
rss.generate_site(token)
# send approval confirmation email to admin
email_body = get_template("approve_comment").render(original=message)
mailer.send(from_email, "Re: " + subject, email_body)

12
app/core/mailer.py
View file

@ -1,3 +1,15 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
def fetch():
pass
def send(email, subject, body):
pass
def delete(content):
# TODO delete mail
pass

160
app/core/processor.py
View file

@ -12,7 +12,7 @@ from threading import Thread
from queue import Queue
from model.site import Site
from model.comment import Comment
from helpers.hashing import md5
from helper.hashing import md5
from conf import config
from core import mailer
@ -26,95 +26,7 @@ env = None
client_ips = {}
class Processor(Thread):
def stop(self):
logger.info("stop requested")
self.is_running = False
def run(self):
logger.info("processor thread started")
self.is_running = True
while self.is_running:
try:
msg = queue.get()
if msg["request"] == "new_mail":
reply_comment_email(msg["data"])
send_delete_command(msg["data"])
else:
logger.info("throw unknown request " + str(msg))
except:
logger.exception("processing failure")
def reply_comment_email(data):
from_email = data["from"]
subject = data["subject"]
message = ""
for part in data["parts"]:
if part["content-type"] == "text/plain":
message = part["content"]
break
m = re.search("\[(\d+)\:(\w+)\]", subject)
if not m:
logger.warn("ignore corrupted email. No token %s" % subject)
return
comment_id = int(m.group(1))
token = m.group(2)
# retrieve site and comment rows
try:
comment = Comment.select().where(Comment.id == comment_id).get()
except:
logger.warn("unknown comment %d" % comment_id)
return
if comment.published:
logger.warn("ignore already published email. token %d" % comment_id)
return
if comment.site.token != token:
logger.warn("ignore corrupted email. Unknown token %d" % comment_id)
return
if not message:
logger.warn("ignore empty email")
return
# safe logic: no answer or unknown answer is a go for publishing
if message[:2].upper() in ("NO", "SP"):
# put a log to help fail2ban
if message[:2].upper() == "SP": # SPAM
if comment_id in client_ips:
logger.info(
"SPAM comment from %s: %d" % (client_ips[comment_id], comment_id)
)
else:
logger.info("cannot identify SPAM source: %d" % comment_id)
# forget client IP
if comment_id in client_ips:
del client_ips[comment_id]
logger.info("discard comment: %d" % comment_id)
comment.delete_instance()
email_body = get_template("drop_comment").render(original=message)
mail(from_email, "Re: " + subject, email_body)
else:
# update Comment row
comment.published = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment.save()
logger.info("commit comment: %d" % comment_id)
# rebuild RSS
rss(token)
# send approval confirmation email to admin
email_body = get_template("approve_comment").render(original=message)
mail(from_email, "Re: " + subject, email_body)
def get_email_metadata(message):
@ -126,73 +38,3 @@ def get_email_metadata(message):
return email
def rss(token, onstart=False):
if onstart and os.path.isfile(config.rss["file"]):
return
site = Site.select().where(Site.token == token).get()
rss_title = get_template("rss_title_message").render(site=site.name)
md = markdown.Markdown()
items = []
for row in (
Comment.select()
.join(Site)
.where(Site.token == token, Comment.published)
.order_by(-Comment.published)
.limit(10)
):
item_link = "%s://%s%s" % (config.rss["proto"], site.url, row.url)
items.append(
PyRSS2Gen.RSSItem(
title="%s - %s://%s%s"
% (config.rss["proto"], row.author_name, site.url, row.url),
link=item_link,
description=md.convert(row.content),
guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)),
pubDate=row.published,
)
)
rss = PyRSS2Gen.RSS2(
title=rss_title,
link="%s://%s" % (config.rss["proto"], site.url),
description="Commentaires du site '%s'" % site.name,
lastBuildDate=datetime.now(),
items=items,
)
rss.write_xml(open(config.rss["file"], "w"), encoding="utf-8")
def send_delete_command(content):
# TODO delete mail
pass
def get_template(name):
return env.get_template(config.general["lang"] + "/" + name + ".tpl")
def enqueue(something):
queue.put(something)
def get_processor():
return proc
def start(template_dir):
global proc, env
# initialize Jinja 2 templating
logger.info("load templates from directory %s" % template_dir)
env = Environment(loader=FileSystemLoader(template_dir))
# generate RSS for all sites
for site in Site.select():
rss(site.token, True)
# start processor thread
proc = Processor()
proc.start()

52
app/core/rss.py Normal file
View file

@ -0,0 +1,52 @@
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from datetime import datetime
import markdown
import PyRSS2Gen
from model.site import Site
from model.comment import Comment
from core.templater import get_template
from conf import config
def generate_all():
for site in Site.select():
generate_site(site.token)
def generate_site(token):
site = Site.select().where(Site.token == token).get()
rss_title = get_template("rss_title_message").render(site=site.name)
md = markdown.Markdown()
items = []
for row in (
Comment.select()
.join(Site)
.where(Site.token == token, Comment.published)
.order_by(-Comment.published)
.limit(10)
):
item_link = "%s://%s%s" % (config.get(config.RSS_PROTO), site.url, row.url)
items.append(
PyRSS2Gen.RSSItem(
title="%s - %s://%s%s"
% (config.get(config.RSS_PROTO), row.author_name, site.url, row.url),
link=item_link,
description=md.convert(row.content),
guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)),
pubDate=row.published,
)
)
rss = PyRSS2Gen.RSS2(
title=rss_title,
link="%s://%s" % (config.get(config.RSS_PROTO), site.url),
description="Commentaires du site '%s'" % site.name,
lastBuildDate=datetime.now(),
items=items,
)
rss.write_xml(open(config.get(config.RSS_FILE), "w"), encoding="utf-8")

3
app/core/templater.py
View file

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
@ -13,4 +12,4 @@ env = Environment(loader=FileSystemLoader(template_path))
def get_template(name):
return env.get_template(config.general["lang"] + "/" + name + ".tpl")
return env.get_template(config.get(config.LANG) + "/" + name + ".tpl")

0
app/helpers/hashing.py → app/helper/hashing.py
View file

2
app/interface/form.py
View file

@ -7,7 +7,7 @@ from flask import request, abort, redirect
from model.site import Site
from model.comment import Comment
from conf import config
from helpers.hashing import md5
from helper.hashing import md5
logger = logging.getLogger(__name__)
app = config.flaskapp()

11
app/run.py
View file

@ -59,9 +59,6 @@ def stacosys_server(config_pathname):
database.setup()
# start processor
from core import processor
# cron email fetcher
app.config.from_object(
JobConfig(
@ -74,10 +71,18 @@ def stacosys_server(config_pathname):
logger.info("Start Stacosys application")
# generate RSS for all sites
from core import rss
rss.generate_all()
# start Flask
from interface import api
from interface import form
logger.debug("Load interface %s" % api)
logger.debug("Load interface %s" % form)
app.run(
host=config.get(config.HTTP_HOST),
port=config.get(config.HTTP_PORT),

7
config.ini
View file

@ -2,6 +2,7 @@
[main]
lang = fr
db_url = sqlite:///db.sqlite
newcomment_polling = 60
[http]
root_url = http://localhost:8100
@ -16,6 +17,6 @@ secret = Uqca5Kc8xuU6THz9
proto = http
file = comments.xml
[polling]
newmail = 15
newcomment = 60
[mail]
fetch_polling = 15
mailer_url = http://localhost:8000