yannic/stacosys
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove site entity

Browse source
This commit is contained in:
Yax 2021-01-10 21:03:55 +01:00
parent dfdb5bdb28
commit acaae53982
11 changed files with 88 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
run.py
View file

@ -7,7 +7,7 @@ import argparse
import logging
from flask import Flask
from stacosys.conf.config import Config, Parameter
from stacosys.conf.config import Config, ConfigParameter
from stacosys.core import database
from stacosys.core.rss import Rss
from stacosys.core.mailer import Mailer
@ -43,45 +43,53 @@ def stacosys_server(config_pathname):
# initialize database
db = database.Database()
db.setup(conf.get(Parameter.DB_URL))
db.setup(conf.get(ConfigParameter.DB_URL))
logger.info("Start Stacosys application")
# generate RSS for all sites
rss = Rss(
conf.get(Parameter.LANG),
conf.get(Parameter.RSS_FILE),
conf.get(Parameter.RSS_PROTO),
conf.get(ConfigParameter.LANG),
conf.get(ConfigParameter.RSS_FILE),
conf.get(ConfigParameter.RSS_PROTO),
conf.get(ConfigParameter.SITE_NAME),
conf.get(ConfigParameter.SITE_URL)
)
rss.generate_all()
rss.generate()
# configure mailer
mailer = Mailer(
conf.get(Parameter.IMAP_HOST),
conf.get_int(Parameter.IMAP_PORT),
conf.get_bool(Parameter.IMAP_SSL),
conf.get(Parameter.IMAP_LOGIN),
conf.get(Parameter.IMAP_PASSWORD),
conf.get(Parameter.SMTP_HOST),
conf.get_int(Parameter.SMTP_PORT),
conf.get_bool(Parameter.SMTP_STARTTLS),
conf.get(Parameter.SMTP_LOGIN),
conf.get(Parameter.SMTP_PASSWORD),
conf.get(ConfigParameter.IMAP_HOST),
conf.get_int(ConfigParameter.IMAP_PORT),
conf.get_bool(ConfigParameter.IMAP_SSL),
conf.get(ConfigParameter.IMAP_LOGIN),
conf.get(ConfigParameter.IMAP_PASSWORD),
conf.get(ConfigParameter.SMTP_HOST),
conf.get_int(ConfigParameter.SMTP_PORT),
conf.get_bool(ConfigParameter.SMTP_STARTTLS),
conf.get(ConfigParameter.SMTP_LOGIN),
conf.get(ConfigParameter.SMTP_PASSWORD),
)
# configure scheduler
scheduler.configure(
conf.get_int(Parameter.IMAP_POLLING),
conf.get_int(Parameter.COMMENT_POLLING),
conf.get(Parameter.LANG),
conf.get_int(ConfigParameter.IMAP_POLLING),
conf.get_int(ConfigParameter.COMMENT_POLLING),
conf.get(ConfigParameter.LANG),
conf.get(ConfigParameter.SITE_NAME),
conf.get(ConfigParameter.SITE_TOKEN),
conf.get(ConfigParameter.SITE_ADMIN_EMAIL),
mailer,
rss,
)
# inject config parameters into flask
app.config.update(SITE_TOKEN=conf.get(ConfigParameter.SITE_TOKEN))
# start Flask
app.run(
host=conf.get(Parameter.HTTP_HOST),
port=conf.get(Parameter.HTTP_PORT),
host=conf.get(ConfigParameter.HTTP_HOST),
port=conf.get(ConfigParameter.HTTP_PORT),
debug=False,
use_reloader=False,
)

2
stacosys/conf/config.py
View file

@ -4,7 +4,7 @@
import profig
class Parameter:
class ConfigParameter:
DB_URL = "main.db_url"
DB_BACKUP_JSON_FILE = "main.db_backup_json_file"
LANG = "main.lang"

12
stacosys/core/cron.py
View file

@ -9,7 +9,7 @@ from datetime import datetime
from stacosys.core import rss
from stacosys.core.templater import Templater, Template
from stacosys.model.comment import Comment, Site
from stacosys.model.comment import Comment
from stacosys.model.email import Email
logger = logging.getLogger(__name__)
@ -18,6 +18,7 @@ current_path = os.path.dirname(__file__)
template_path = os.path.abspath(os.path.join(current_path, "../templates"))
templater = Templater(template_path)
def fetch_mail_answers(lang, mailer, rss):
for msg in mailer.fetch():
if re.search(r".*STACOSYS.*\[(\d+)\:(\w+)\]", msg.subject, re.DOTALL):
@ -36,7 +37,7 @@ def _reply_comment_email(lang, mailer, rss, email: Email):
# retrieve site and comment rows
try:
comment = Comment.select().where(Comment.id == comment_id).get()
comment = Comment.get_by_id(comment_id)
except:
logger.warn("unknown comment %d" % comment_id)
return True
@ -80,7 +81,7 @@ def _reply_comment_email(lang, mailer, rss, email: Email):
return True
def submit_new_comment(lang, mailer):
def submit_new_comment(lang, site_name, site_token, site_admin_email, mailer):
for comment in Comment.select().where(Comment.notified.is_null()):
@ -99,9 +100,8 @@ def submit_new_comment(lang, mailer):
)
# send email
site = Site.get(Site.id == comment.site)
subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token)
if mailer.send(site.admin_email, subject, email_body):
subject = "STACOSYS %s: [%d:%s]" % (site_name, comment.id, site_token)
if mailer.send(site_admin_email, subject, email_body):
logger.debug("new comment processed ")
# notify site admin and save notification datetime

4
stacosys/core/database.py
View file

@ -6,7 +6,6 @@ from peewee import DatabaseProxy, Model
from playhouse.db_url import connect, SqliteDatabase
from playhouse.shortcuts import model_to_dict
from tinydb import TinyDB
from stacosys.conf import config
db = SqliteDatabase(None)
@ -25,10 +24,9 @@ class Database:
db.init(db_url)
db.connect()
from stacosys.model.site import Site
from stacosys.model.comment import Comment
db.create_tables([Site, Comment], safe=True)
db.create_tables([Comment], safe=True)
# if config.exists(config.DB_BACKUP_JSON_FILE):

29
stacosys/core/rss.py
View file

@ -10,44 +10,39 @@ import PyRSS2Gen
import stacosys.conf.config as config
from stacosys.core.templater import Templater, Template
from stacosys.model.comment import Comment
from stacosys.model.site import Site
class Rss:
def __init__(self, lang, rss_file, rss_proto):
def __init__(
self, lang, rss_file, rss_proto, site_name, site_url,
):
self._lang = lang
self._rss_file = rss_file
self._rss_proto = rss_proto
self._site_name = site_name
self._site_url = site_url
current_path = os.path.dirname(__file__)
template_path = os.path.abspath(os.path.join(current_path, "../templates"))
self._templater = Templater(template_path)
def generate_all(self):
for site in Site.select():
self._generate_site(site.token)
def _generate_site(self, token):
site = Site.select().where(Site.token == token).get()
def generate(self):
rss_title = self._templater.get_template(
self._lang, Template.RSS_TITLE_MESSAGE
).render(site=site.name)
).render(site=self._site_name)
md = markdown.Markdown()
items = []
for row in (
Comment.select()
.join(Site)
.where(Site.token == token, Comment.published)
.where(Comment.published)
.order_by(-Comment.published)
.limit(10)
):
item_link = "%s://%s%s" % (self._rss_proto, site.url, row.url)
item_link = "%s://%s%s" % (self._rss_proto, self._site_url, row.url)
items.append(
PyRSS2Gen.RSSItem(
title="%s - %s://%s%s"
% (self._rss_proto, row.author_name, site.url, row.url),
% (self._rss_proto, row.author_name, self._site_url, row.url),
link=item_link,
description=md.convert(row.content),
guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)),
@ -57,8 +52,8 @@ class Rss:
rss = PyRSS2Gen.RSS2(
title=rss_title,
link="%s://%s" % (self._rss_proto, site.url),
description='Commentaires du site "%s"' % site.name,
link="%s://%s" % (self._rss_proto, self._site_url),
description='Commentaires du site "%s"' % self._site_name,
lastBuildDate=datetime.now(),
items=items,
)

22
stacosys/interface/api.py
View file

@ -4,9 +4,9 @@
import logging
from flask import abort, jsonify, request
from stacosys.conf.config import ConfigParameter
from stacosys.interface import app
from stacosys.model.comment import Comment
from stacosys.model.site import Site
logger = logging.getLogger(__name__)
@ -22,17 +22,15 @@ def query_comments():
comments = []
try:
token = request.args.get("token", "")
if token != app.config.get(ConfigParameter.SITE_TOKEN):
abort(401)
url = request.args.get("url", "")
logger.info("retrieve comments for url %s" % (url))
for comment in (
Comment.select(Comment)
.join(Site)
.where(
(Comment.url == url)
& (Comment.published.is_null(False))
& (Site.token == token)
)
.where((Comment.url == url) & (Comment.published.is_null(False)))
.order_by(+Comment.published)
):
d = {
@ -58,15 +56,13 @@ def query_comments():
def get_comments_count():
try:
token = request.args.get("token", "")
if token != app.config.get(ConfigParameter.SITE_TOKEN):
abort(401)
url = request.args.get("url", "")
count = (
Comment.select(Comment)
.join(Site)
.where(
(Comment.url == url)
& (Comment.published.is_null(False))
& (Site.token == token)
)
.where((Comment.url == url) & (Comment.published.is_null(False)))
.count()
)
r = jsonify({"count": count})

9
stacosys/interface/form.py
View file

@ -5,9 +5,9 @@ import logging
from datetime import datetime
from flask import abort, redirect, request
from stacosys.conf.config import ConfigParameter
from stacosys.interface import app
from stacosys.model.comment import Comment
from stacosys.model.site import Site
logger = logging.getLogger(__name__)
@ -21,10 +21,8 @@ def new_form_comment():
# validate token: retrieve site entity
token = data.get("token", "")
site = Site.select().where(Site.token == token).get()
if site is None:
logger.warn("Unknown site %s" % token)
abort(400)
if token != app.config.get(ConfigParameter.SITE_TOKEN):
abort(401)
# honeypot for spammers
captcha = data.get("remarque", "")
@ -49,7 +47,6 @@ def new_form_comment():
# add a row to Comment table
created = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment = Comment(
site=site,
url=url,
author_name=author_name,
author_site=author_site,

8
stacosys/interface/scheduler.py
View file

@ -11,7 +11,7 @@ class JobConfig(object):
SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 4}}
def __init__(self, imap_polling_seconds, new_comment_polling_seconds, lang, mailer, rss):
def __init__(self, imap_polling_seconds, new_comment_polling_seconds, lang, site_name, site_token, site_admin_email, mailer, rss):
self.JOBS = [
{
"id": "fetch_mail",
@ -23,15 +23,15 @@ class JobConfig(object):
{
"id": "submit_new_comment",
"func": "stacosys.core.cron:submit_new_comment",
"args": [lang, mailer],
"args": [lang, site_name, site_token, site_admin_email, mailer],
"trigger": "interval",
"seconds": new_comment_polling_seconds,
},
]
def configure(imap_polling, comment_polling, lang, mailer, rss):
app.config.from_object(JobConfig(imap_polling, comment_polling, lang, mailer, rss))
def configure(imap_polling, comment_polling, lang, site_name, site_token, site_admin_email, mailer, rss):
app.config.from_object(JobConfig(imap_polling, comment_polling, lang, site_name, site_token, site_admin_email, mailer, rss))
scheduler = APScheduler()
scheduler.init_app(app)
scheduler.start()

3
stacosys/model/comment.py
View file

@ -6,10 +6,10 @@ from peewee import CharField
from peewee import TextField
from peewee import DateTimeField
from peewee import ForeignKeyField
from stacosys.model.site import Site
from datetime import datetime
from stacosys.core.database import BaseModel
class Comment(BaseModel):
url = CharField()
created = DateTimeField()
@ -19,7 +19,6 @@ class Comment(BaseModel):
author_site = CharField(default="")
author_gravatar = CharField(default="")
content = TextField()
site = ForeignKeyField(Site, related_name="site")
def notify_site_admin(self):
self.notified = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

13
stacosys/model/site.py
View file

@ -1,13 +0,0 @@
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from peewee import Model
from peewee import CharField
from stacosys.core.database import BaseModel
class Site(BaseModel):
name = CharField(unique=True)
url = CharField()
token = CharField()
admin_email = CharField()

44
tests/test_config.py
View file

@ -2,7 +2,7 @@
# -*- coding: UTF-8 -*-
import pytest
from stacosys.conf.config import Config, Parameter
from stacosys.conf.config import Config, ConfigParameter
EXPECTED_DB_URL = "sqlite:///db.sqlite"
EXPECTED_HTTP_PORT = 8080
@ -13,37 +13,37 @@ EXPECTED_IMAP_LOGIN = "user"
@pytest.fixture
def conf():
conf = Config()
conf.put(Parameter.DB_URL, EXPECTED_DB_URL)
conf.put(Parameter.HTTP_PORT, EXPECTED_HTTP_PORT)
conf.put(Parameter.IMAP_PORT, EXPECTED_IMAP_PORT)
conf.put(Parameter.SMTP_STARTTLS, "yes")
conf.put(Parameter.IMAP_SSL, "false")
conf.put(ConfigParameter.DB_URL, EXPECTED_DB_URL)
conf.put(ConfigParameter.HTTP_PORT, EXPECTED_HTTP_PORT)
conf.put(ConfigParameter.IMAP_PORT, EXPECTED_IMAP_PORT)
conf.put(ConfigParameter.SMTP_STARTTLS, "yes")
conf.put(ConfigParameter.IMAP_SSL, "false")
return conf
def test_exists(conf):
assert conf is not None
assert conf.exists(Parameter.DB_URL)
assert not conf.exists(Parameter.IMAP_HOST)
assert conf.exists(ConfigParameter.DB_URL)
assert not conf.exists(ConfigParameter.IMAP_HOST)
def test_get(conf):
assert conf is not None
assert conf.get(Parameter.DB_URL) == EXPECTED_DB_URL
assert conf.get(Parameter.HTTP_PORT) == EXPECTED_HTTP_PORT
assert conf.get(Parameter.HTTP_HOST) is None
assert conf.get(Parameter.HTTP_PORT) == EXPECTED_HTTP_PORT
assert conf.get(Parameter.IMAP_PORT) == EXPECTED_IMAP_PORT
assert conf.get_int(Parameter.IMAP_PORT) == int(EXPECTED_IMAP_PORT)
assert conf.get(ConfigParameter.DB_URL) == EXPECTED_DB_URL
assert conf.get(ConfigParameter.HTTP_PORT) == EXPECTED_HTTP_PORT
assert conf.get(ConfigParameter.HTTP_HOST) is None
assert conf.get(ConfigParameter.HTTP_PORT) == EXPECTED_HTTP_PORT
assert conf.get(ConfigParameter.IMAP_PORT) == EXPECTED_IMAP_PORT
assert conf.get_int(ConfigParameter.IMAP_PORT) == int(EXPECTED_IMAP_PORT)
try:
conf.get_int(Parameter.HTTP_PORT)
conf.get_int(ConfigParameter.HTTP_PORT)
assert False
except:
pass
assert conf.get_bool(Parameter.SMTP_STARTTLS)
assert not conf.get_bool(Parameter.IMAP_SSL)
assert conf.get_bool(ConfigParameter.SMTP_STARTTLS)
assert not conf.get_bool(ConfigParameter.IMAP_SSL)
try:
conf.get_bool(Parameter.DB_URL)
conf.get_bool(ConfigParameter.DB_URL)
assert False
except:
pass
@ -51,7 +51,7 @@ def test_get(conf):
def test_put(conf):
assert conf is not None
assert not conf.exists(Parameter.IMAP_LOGIN)
conf.put(Parameter.IMAP_LOGIN, EXPECTED_IMAP_LOGIN)
assert conf.exists(Parameter.IMAP_LOGIN)
assert conf.get(Parameter.IMAP_LOGIN) == EXPECTED_IMAP_LOGIN
assert not conf.exists(ConfigParameter.IMAP_LOGIN)
conf.put(ConfigParameter.IMAP_LOGIN, EXPECTED_IMAP_LOGIN)
assert conf.exists(ConfigParameter.IMAP_LOGIN)
assert conf.get(ConfigParameter.IMAP_LOGIN) == EXPECTED_IMAP_LOGIN