Stacosys 1.1.0 - Poetry

stacosys/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.1.0'

stacosys/conf/config.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import profig
+
+# constants
+FLASK_APP = "flask.app"
+
+DB_URL = "main.db_url"
+LANG = "main.lang"
+
+HTTP_HOST = "http.host"
+HTTP_PORT = "http.port"
+
+SECURITY_SALT = "security.salt"
+SECURITY_SECRET = "security.secret"
+
+RSS_PROTO = "rss.proto"
+RSS_FILE = "rss.file"
+
+MAIL_POLLING = "mail.fetch_polling"
+COMMENT_POLLING = "main.newcomment_polling"
+MAILER_URL = "mail.mailer_url"
+
+
+# variable
+params = dict()
+
+
+def initialize(config_pathname, flask_app):
+    cfg = profig.Config(config_pathname)
+    cfg.sync()
+    params.update(cfg)
+    params.update({FLASK_APP: flask_app})
+
+
+def get(key):
+    return params[key]
+
+
+def getInt(key):
+    return int(params[key])
+
+
+def _str2bool(v):
+    return v.lower() in ("yes", "true", "t", "1")
+
+
+def getBool(key):
+    return _str2bool(params[key])
+
+
+def flaskapp():
+    return params[FLASK_APP]

stacosys/core/cron.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import logging
+from datetime import datetime
+import time
+import re
+from core import mailer
+from core.templater import get_template
+from core import rss
+from model.comment import Comment
+from model.comment import Site
+
+logger = logging.getLogger(__name__)
+
+
+def cron(func):
+    def wrapper():
+        logger.debug("execute CRON " + func.__name__)
+        func()
+
+    return wrapper
+
+
+@cron
+def fetch_mail_answers():
+
+    for msg in mailer.fetch():
+        if re.search(r".*STACOSYS.*\[(\d+)\:(\w+)\]", msg["subject"], re.DOTALL):
+            full_msg = mailer.get(msg["id"])
+            if full_msg and reply_comment_email(full_msg['email']):
+                mailer.delete(msg["id"])
+
+@cron
+def submit_new_comment():
+
+    for comment in Comment.select().where(Comment.notified.is_null()):
+
+        comment_list = (
+            "author: %s" % comment.author_name,
+            "site: %s" % comment.author_site,
+            "date: %s" % comment.created,
+            "url: %s" % comment.url,
+            "",
+            "%s" % comment.content,
+            "",
+        )
+        comment_text = "\n".join(comment_list)
+        email_body = get_template("new_comment").render(
+            url=comment.url, comment=comment_text
+        )
+
+        # send email
+        site = Site.get(Site.id == comment.site)
+        subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token)
+        mailer.send(site.admin_email, subject, email_body)
+        logger.debug("new comment processed ")
+
+        # update comment
+        comment.notified = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        comment.save()
+
+
+def reply_comment_email(data):
+
+    from_email = data["from"]
+    subject = data["subject"]
+    message = ""
+    for part in data["parts"]:
+        if part["content-type"] == "text/plain":
+            message = part["content"]
+            break
+
+    m = re.search(r"\[(\d+)\:(\w+)\]", subject)
+    if not m:
+        logger.warn("ignore corrupted email. No token %s" % subject)
+        return
+    comment_id = int(m.group(1))
+    token = m.group(2)
+
+    # retrieve site and comment rows
+    try:
+        comment = Comment.select().where(Comment.id == comment_id).get()
+    except:
+        logger.warn("unknown comment %d" % comment_id)
+        return True
+
+    if comment.published:
+        logger.warn("ignore already published email. token %d" % comment_id)
+        return
+
+    if comment.site.token != token:
+        logger.warn("ignore corrupted email. Unknown token %d" % comment_id)
+        return
+
+    if not message:
+        logger.warn("ignore empty email")
+        return
+
+    # safe logic: no answer or unknown answer is a go for publishing
+    if message[:2].upper() in ("NO", "SP"):
+
+        # put a log to help fail2ban
+        if message[:2].upper() == "SP":  # SPAM
+            if comment.ip:
+                logger.info(
+                    "SPAM comment from %s: %d" % (comment.ip, comment_id)
+                )
+            else:
+                logger.info("cannot identify SPAM source: %d" % comment_id)
+
+        logger.info("discard comment: %d" % comment_id)
+        comment.delete_instance()
+        email_body = get_template("drop_comment").render(original=message)
+        mailer.send(from_email, "Re: " + subject, email_body)
+    else:
+        # update Comment row
+        comment.published = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        comment.ip = None
+        comment.save()
+        logger.info("commit comment: %d" % comment_id)
+
+        # rebuild RSS
+        rss.generate_site(token)
+
+        # send approval confirmation email to admin
+        email_body = get_template("approve_comment").render(original=message)
+        mailer.send(from_email, "Re: " + subject, email_body)
+
+    return True

stacosys/core/database.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+from conf import config
+from playhouse.db_url import connect
+
+
+def get_db():
+    return connect(config.get(config.DB_URL))
+
+
+def setup():
+    from model.site import Site
+    from model.comment import Comment
+
+    get_db().create_tables([Site, Comment], safe=True)

stacosys/core/mailer.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import logging
+import json
+import requests
+from conf import config
+
+logger = logging.getLogger(__name__)
+
+
+def fetch():
+    mails = []
+    r = requests.get(config.get(config.MAILER_URL) + "/mbox")
+    if r.status_code == 200:
+        payload = r.json()
+        if payload["count"] > 0:
+            mails = payload["emails"]
+    return mails
+
+
+def get(id):
+    payload = None
+    r = requests.get(config.get(config.MAILER_URL) + "/mbox/" + str(id))
+    if r.status_code == 200:
+        payload = r.json()
+    return payload
+
+
+def send(to_email, subject, message):
+    headers = {"Content-Type": "application/json; charset=utf-8"}
+    msg = {"to": to_email, "subject": subject, "content": message}
+    r = requests.post(
+        config.get(config.MAILER_URL) + "/mbox", data=json.dumps(msg), headers=headers
+    )
+    if r.status_code in (200, 201):
+        logger.debug("Email for %s posted" % to_email)
+    else:
+        logger.warn("Cannot post email for %s" % to_email)
+
+
+def delete(id):
+    requests.delete(config.get(config.MAILER_URL) + "/mbox/" + str(id))

stacosys/core/rss.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+from datetime import datetime
+import markdown
+import PyRSS2Gen
+from model.site import Site
+from model.comment import Comment
+from core.templater import get_template
+from conf import config
+
+
+def generate_all():
+    for site in Site.select():
+        generate_site(site.token)
+
+
+def generate_site(token):
+
+    site = Site.select().where(Site.token == token).get()
+    rss_title = get_template("rss_title_message").render(site=site.name)
+    md = markdown.Markdown()
+
+    items = []
+    for row in (
+        Comment.select()
+        .join(Site)
+        .where(Site.token == token, Comment.published)
+        .order_by(-Comment.published)
+        .limit(10)
+    ):
+        item_link = "%s://%s%s" % (config.get(config.RSS_PROTO), site.url, row.url)
+        items.append(
+            PyRSS2Gen.RSSItem(
+                title="%s - %s://%s%s"
+                % (config.get(config.RSS_PROTO), row.author_name, site.url, row.url),
+                link=item_link,
+                description=md.convert(row.content),
+                guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)),
+                pubDate=row.published,
+            )
+        )
+
+    rss = PyRSS2Gen.RSS2(
+        title=rss_title,
+        link="%s://%s" % (config.get(config.RSS_PROTO), site.url),
+        description="Commentaires du site '%s'" % site.name,
+        lastBuildDate=datetime.now(),
+        items=items,
+    )
+    rss.write_xml(open(config.get(config.RSS_FILE), "w"), encoding="utf-8")
+

stacosys/core/templater.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+from jinja2 import Environment
+from jinja2 import FileSystemLoader
+from conf import config
+
+current_path = os.path.dirname(__file__)
+template_path = os.path.abspath(os.path.join(current_path, "../templates"))
+env = Environment(loader=FileSystemLoader(template_path))
+
+
+def get_template(name):
+    return env.get_template(config.get(config.LANG) + "/" + name + ".tpl")

stacosys/helper/hashing.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+import hashlib
+from conf import config
+
+
+def salt(value):
+    string = "%s%s" % (value, config.get(config.SECURITY_SALT))
+    dk = hashlib.sha256(string.encode())
+    return dk.hexdigest()
+
+
+def md5(value):
+    dk = hashlib.md5(value.encode())
+    return dk.hexdigest()

stacosys/interface/api.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import logging
+from flask import request, jsonify, abort
+from model.site import Site
+from model.comment import Comment
+from conf import config
+
+logger = logging.getLogger(__name__)
+app = config.flaskapp()
+
+@app.route("/ping", methods=['GET'])
+def ping():
+    return "OK"
+
+
+@app.route("/comments", methods=['GET'])
+def query_comments():
+
+    comments = []
+    try:
+        token = request.args.get('token', '')
+        url = request.args.get('url', '')
+
+        logger.info('retrieve comments for token %s, url %s' % (token, url))
+        for comment in Comment.select(Comment).join(Site).where(
+                (Comment.url == url) &
+                (Comment.published.is_null(False)) &
+                (Site.token == token)).order_by(+Comment.published):
+            d = {}
+            d['author'] = comment.author_name
+            d['content'] = comment.content
+            if comment.author_site:
+                d['site'] = comment.author_site
+            d['avatar'] = comment.author_gravatar
+            d['date'] = comment.published.strftime("%Y-%m-%d %H:%M:%S")
+            logger.debug(d)
+            comments.append(d)
+        r = jsonify({'data': comments})
+        r.status_code = 200
+    except:
+        logger.warn('bad request')
+        r = jsonify({'data': []})
+        r.status_code = 400
+    return r
+
+
+@app.route("/comments/count", methods=['GET'])
+def get_comments_count():
+    try:
+        token = request.args.get('token', '')
+        url = request.args.get('url', '')
+        count = Comment.select(Comment).join(Site).where(
+            (Comment.url == url) &
+            (Comment.published.is_null(False)) &
+            (Site.token == token)).count()
+        r = jsonify({'count': count})
+        r.status_code = 200
+    except:
+        r = jsonify({'count': 0})
+        r.status_code = 200
+    return r
+
+
+@app.route("/comments", methods=['POST'])
+def new_comment():
+
+    try:
+        data = request.get_json()
+        logger.info(data)
+
+        # validate token: retrieve site entity
+        token = data.get('token', '')
+        site = Site.select().where(Site.token == token).get()
+        if site is None:
+            logger.warn('Unknown site %s' % token)
+            abort(400)
+
+        # honeypot for spammers
+        captcha = data.get('captcha', '')
+        if captcha:
+            logger.warn('discard spam: data %s' % data)
+            abort(400)
+
+        processor.enqueue({'request': 'new_comment', 'data': data})
+
+    except:
+        logger.exception("new comment failure")
+        abort(400)
+
+    return "OK"

stacosys/interface/form.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import logging
+from datetime import datetime
+from flask import request, abort, redirect
+from model.site import Site
+from model.comment import Comment
+from conf import config
+from helper.hashing import md5
+
+logger = logging.getLogger(__name__)
+app = config.flaskapp()
+
+@app.route("/newcomment", methods=["POST"])
+def new_form_comment():
+
+    try:
+        data = request.form
+        logger.info("form data " + str(data))
+
+        # add client IP if provided by HTTP proxy
+        ip = ""
+        if "X-Forwarded-For" in request.headers:
+            ip = request.headers["X-Forwarded-For"]
+
+        # validate token: retrieve site entity
+        token = data.get("token", "")
+        site = Site.select().where(Site.token == token).get()
+        if site is None:
+            logger.warn("Unknown site %s" % token)
+            abort(400)
+
+        # honeypot for spammers
+        captcha = data.get("remarque", "")        
+        if captcha:
+            logger.warn("discard spam: data %s" % data)
+            abort(400)
+
+        url = data.get("url", "") 
+        author_name = data.get("author", "").strip()
+        author_gravatar = data.get("email", "").strip()
+        author_site = data.get("site", "").lower().strip()
+        if author_site and author_site[:4] != "http":
+            author_site = "http://" + author_site
+        message = data.get("message", "")
+
+        # anti-spam again
+        if not url or not author_name or not message:
+            logger.warn("empty field: data %s" % data)
+            abort(400)
+        check_form_data(data)
+
+        # add a row to Comment table
+        created = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        comment = Comment(
+            site=site,
+            url=url,
+            author_name=author_name,
+            author_site=author_site,
+            author_gravatar=author_gravatar,
+            content=message,
+            created=created,
+            notified=None,
+            published=None,
+            ip=ip,
+        )
+        comment.save()
+
+    except:
+        logger.exception("new comment failure")
+        abort(400)
+
+    return redirect("/redirect/", code=302)
+
+def check_form_data(data):
+    fields = ['url', 'message', 'site', 'remarque', 'author', 'token', 'email']
+    d = data.to_dict()
+    for field in fields:
+        if field in d:
+            del d[field]
+    if d: 
+        logger.warn("additional field: data %s" % data)
+        abort(400)

stacosys/model/comment.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+from peewee import Model
+from peewee import CharField
+from peewee import TextField
+from peewee import DateTimeField
+from peewee import ForeignKeyField
+from model.site import Site
+from core.database import get_db
+
+
+class Comment(Model):
+    url = CharField()
+    created = DateTimeField()
+    notified = DateTimeField(null=True,default=None)
+    published = DateTimeField(null=True, default=None)
+    author_name = CharField()
+    author_site = CharField(default='')
+    author_gravatar = CharField(default='')
+    ip = CharField(default='')
+    content = TextField()
+    site = ForeignKeyField(Site, related_name='site')
+
+    class Meta:
+        database = get_db()

stacosys/model/site.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+from peewee import Model
+from peewee import CharField
+from core.database import get_db
+
+
+class Site(Model):
+    name = CharField(unique=True)
+    url = CharField()
+    token = CharField()
+    admin_email = CharField()
+
+    class Meta:
+        database = get_db()

stacosys/run.py

@@ -0,0 +1,100 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+import argparse
+import os
+import logging
+from flask import Flask
+from flask_apscheduler import APScheduler
+from conf import config
+
+# configure logging
+def configure_logging(level):
+    root_logger = logging.getLogger()
+    root_logger.setLevel(level)
+    ch = logging.StreamHandler()
+    ch.setLevel(level)
+    # create formatter
+    formatter = logging.Formatter("[%(asctime)s] %(name)s %(levelname)s %(message)s")
+    # add formatter to ch
+    ch.setFormatter(formatter)
+    # add ch to logger
+    root_logger.addHandler(ch)
+
+
+class JobConfig(object):
+
+    JOBS = []
+
+    SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 20}}
+
+    def __init__(self, mail_polling_seconds, new_comment_polling_seconds):
+        self.JOBS = [
+            {
+                "id": "fetch_mail",
+                "func": "core.cron:fetch_mail_answers",
+                "trigger": "interval",
+                "seconds": mail_polling_seconds,
+            },
+            {
+                "id": "submit_new_comment",
+                "func": "core.cron:submit_new_comment",
+                "trigger": "interval",
+                "seconds": new_comment_polling_seconds,
+            },
+        ]
+
+
+def stacosys_server(config_pathname):
+
+    app = Flask(__name__)
+    config.initialize(config_pathname, app)
+
+    # configure logging
+    logger = logging.getLogger(__name__)
+    configure_logging(logging.INFO)
+    logging.getLogger("werkzeug").level = logging.WARNING
+    logging.getLogger("apscheduler.executors").level = logging.WARNING
+
+    # initialize database
+    from core import database
+
+    database.setup()
+
+    # cron email fetcher
+    app.config.from_object(
+        JobConfig(
+            config.getInt(config.MAIL_POLLING), config.getInt(config.COMMENT_POLLING)
+        )
+    )
+    scheduler = APScheduler()
+    scheduler.init_app(app)
+    scheduler.start()
+
+    logger.info("Start Stacosys application")
+
+    # generate RSS for all sites
+    from core import rss
+
+    rss.generate_all()
+
+    # start Flask
+    from interface import api
+    from interface import form
+
+    logger.debug("Load interface %s" % api)
+    logger.debug("Load interface %s" % form)
+
+    app.run(
+        host=config.get(config.HTTP_HOST),
+        port=config.get(config.HTTP_PORT),
+        debug=False,
+        use_reloader=False,
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("config", help="config path name")
+    args = parser.parse_args()
+    stacosys_server(args.config)

stacosys/templates/en/approve_comment.tpl

@@ -0,0 +1,9 @@
+Hi,
+
+The comment should be published soon. It has been approved.
+
+--
+Stacosys
+
+
+{{ original }}

stacosys/templates/en/drop_comment.tpl

@@ -0,0 +1,9 @@
+Hi,
+
+The comment will not be published. It has been dropped.
+
+--
+Stacosys
+
+
+{{ original }}

stacosys/templates/en/new_comment.tpl

@@ -0,0 +1,16 @@
+Hi,
+
+A new comment has been submitted for post {{ url }}
+
+You have two choices:
+- reject the comment by replying NO (or no),
+- accept the comment by sending back the email as it is.
+
+If you choose the latter option, Stacosys is going to publish the commennt.
+
+Please find comment details below:
+
+{{ comment }}
+
+--
+Stacosys

stacosys/templates/en/notify_message.tpl

@@ -0,0 +1 @@
+New comment

stacosys/templates/en/rss_title_message.tpl

@@ -0,0 +1 @@
+{{ site }} : comments

stacosys/templates/fr/approve_comment.tpl

@@ -0,0 +1,9 @@
+Bonjour,
+
+Le commentaire sera bientôt publié. Il a été approuvé.
+
+--
+Stacosys
+
+
+{{ original }}

stacosys/templates/fr/drop_comment.tpl

@@ -0,0 +1,9 @@
+Bonjour,
+
+Le commentaire ne sera pas publié. Il a été rejeté.
+
+--
+Stacosys
+
+
+{{ original }}

stacosys/templates/fr/new_comment.tpl

@@ -0,0 +1,16 @@
+Bonjour,
+
+Un nouveau commentaire a été posté pour l'article {{ url }}
+
+Vous avez deux réponses possibles :
+- rejeter le commentaire en répondant NO (ou no),
+- accepter le commentaire en renvoyant cet email tel quel.
+
+Si cette dernière option est choisie, Stacosys publiera le commentaire très bientôt.
+
+Voici les détails concernant le commentaire :
+
+{{ comment }}
+
+--
+Stacosys

stacosys/templates/fr/notify_message.tpl

@@ -0,0 +1 @@
+Nouveau commentaire

stacosys/templates/fr/rss_title_message.tpl

@@ -0,0 +1 @@
+{{ site }} : commentaires