yannic/stacosys
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove site entity

Browse source
This commit is contained in:
Yax 2021-01-10 21:03:55 +01:00
parent e85e53a48e
commit e6986e3587
11 changed files with 88 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
run.py
View file

@ -7,7 +7,7 @@ import argparse
import logging import logging
from flask import Flask from flask import Flask
from stacosys.conf.config import Config, Parameter from stacosys.conf.config import Config, ConfigParameter
from stacosys.core import database from stacosys.core import database
from stacosys.core.rss import Rss from stacosys.core.rss import Rss
from stacosys.core.mailer import Mailer from stacosys.core.mailer import Mailer
@ -43,45 +43,53 @@ def stacosys_server(config_pathname):
# initialize database # initialize database
db = database.Database() db = database.Database()
db.setup(conf.get(Parameter.DB_URL)) db.setup(conf.get(ConfigParameter.DB_URL))
logger.info("Start Stacosys application") logger.info("Start Stacosys application")
# generate RSS for all sites # generate RSS for all sites
rss = Rss( rss = Rss(
conf.get(Parameter.LANG), conf.get(ConfigParameter.LANG),
conf.get(Parameter.RSS_FILE), conf.get(ConfigParameter.RSS_FILE),
conf.get(Parameter.RSS_PROTO), conf.get(ConfigParameter.RSS_PROTO),
conf.get(ConfigParameter.SITE_NAME),
conf.get(ConfigParameter.SITE_URL)
) )
rss.generate_all() rss.generate()
# configure mailer # configure mailer
mailer = Mailer( mailer = Mailer(
conf.get(Parameter.IMAP_HOST), conf.get(ConfigParameter.IMAP_HOST),
conf.get_int(Parameter.IMAP_PORT), conf.get_int(ConfigParameter.IMAP_PORT),
conf.get_bool(Parameter.IMAP_SSL), conf.get_bool(ConfigParameter.IMAP_SSL),
conf.get(Parameter.IMAP_LOGIN), conf.get(ConfigParameter.IMAP_LOGIN),
conf.get(Parameter.IMAP_PASSWORD), conf.get(ConfigParameter.IMAP_PASSWORD),
conf.get(Parameter.SMTP_HOST), conf.get(ConfigParameter.SMTP_HOST),
conf.get_int(Parameter.SMTP_PORT), conf.get_int(ConfigParameter.SMTP_PORT),
conf.get_bool(Parameter.SMTP_STARTTLS), conf.get_bool(ConfigParameter.SMTP_STARTTLS),
conf.get(Parameter.SMTP_LOGIN), conf.get(ConfigParameter.SMTP_LOGIN),
conf.get(Parameter.SMTP_PASSWORD), conf.get(ConfigParameter.SMTP_PASSWORD),
) )
# configure scheduler # configure scheduler
scheduler.configure( scheduler.configure(
conf.get_int(Parameter.IMAP_POLLING), conf.get_int(ConfigParameter.IMAP_POLLING),
conf.get_int(Parameter.COMMENT_POLLING), conf.get_int(ConfigParameter.COMMENT_POLLING),
conf.get(Parameter.LANG), conf.get(ConfigParameter.LANG),
conf.get(ConfigParameter.SITE_NAME),
conf.get(ConfigParameter.SITE_TOKEN),
conf.get(ConfigParameter.SITE_ADMIN_EMAIL),
mailer, mailer,
rss, rss,
) )
# inject config parameters into flask
app.config.update(SITE_TOKEN=conf.get(ConfigParameter.SITE_TOKEN))
# start Flask # start Flask
app.run( app.run(
host=conf.get(Parameter.HTTP_HOST), host=conf.get(ConfigParameter.HTTP_HOST),
port=conf.get(Parameter.HTTP_PORT), port=conf.get(ConfigParameter.HTTP_PORT),
debug=False, debug=False,
use_reloader=False, use_reloader=False,
) )

2
stacosys/conf/config.py
View file

@ -4,7 +4,7 @@
import profig import profig
class Parameter: class ConfigParameter:
DB_URL = "main.db_url" DB_URL = "main.db_url"
DB_BACKUP_JSON_FILE = "main.db_backup_json_file" DB_BACKUP_JSON_FILE = "main.db_backup_json_file"
LANG = "main.lang" LANG = "main.lang"

12
stacosys/core/cron.py
View file

@ -9,7 +9,7 @@ from datetime import datetime
from stacosys.core import rss from stacosys.core import rss
from stacosys.core.templater import Templater, Template from stacosys.core.templater import Templater, Template
from stacosys.model.comment import Comment, Site from stacosys.model.comment import Comment
from stacosys.model.email import Email from stacosys.model.email import Email
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -18,6 +18,7 @@ current_path = os.path.dirname(__file__)
template_path = os.path.abspath(os.path.join(current_path, "../templates")) template_path = os.path.abspath(os.path.join(current_path, "../templates"))
templater = Templater(template_path) templater = Templater(template_path)
def fetch_mail_answers(lang, mailer, rss): def fetch_mail_answers(lang, mailer, rss):
for msg in mailer.fetch(): for msg in mailer.fetch():
if re.search(r".*STACOSYS.*\[(\d+)\:(\w+)\]", msg.subject, re.DOTALL): if re.search(r".*STACOSYS.*\[(\d+)\:(\w+)\]", msg.subject, re.DOTALL):
@ -36,7 +37,7 @@ def _reply_comment_email(lang, mailer, rss, email: Email):
# retrieve site and comment rows # retrieve site and comment rows
try: try:
comment = Comment.select().where(Comment.id == comment_id).get() comment = Comment.get_by_id(comment_id)
except: except:
logger.warn("unknown comment %d" % comment_id) logger.warn("unknown comment %d" % comment_id)
return True return True
@ -80,7 +81,7 @@ def _reply_comment_email(lang, mailer, rss, email: Email):
return True return True
def submit_new_comment(lang, mailer): def submit_new_comment(lang, site_name, site_token, site_admin_email, mailer):
for comment in Comment.select().where(Comment.notified.is_null()): for comment in Comment.select().where(Comment.notified.is_null()):
@ -99,9 +100,8 @@ def submit_new_comment(lang, mailer):
) )
# send email # send email
site = Site.get(Site.id == comment.site) subject = "STACOSYS %s: [%d:%s]" % (site_name, comment.id, site_token)
subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token) if mailer.send(site_admin_email, subject, email_body):
if mailer.send(site.admin_email, subject, email_body):
logger.debug("new comment processed ") logger.debug("new comment processed ")
# notify site admin and save notification datetime # notify site admin and save notification datetime

4
stacosys/core/database.py
View file

@ -6,7 +6,6 @@ from peewee import DatabaseProxy, Model
from playhouse.db_url import connect, SqliteDatabase from playhouse.db_url import connect, SqliteDatabase
from playhouse.shortcuts import model_to_dict from playhouse.shortcuts import model_to_dict
from tinydb import TinyDB from tinydb import TinyDB
from stacosys.conf import config
db = SqliteDatabase(None) db = SqliteDatabase(None)
@ -25,10 +24,9 @@ class Database:
db.init(db_url) db.init(db_url)
db.connect() db.connect()
from stacosys.model.site import Site
from stacosys.model.comment import Comment from stacosys.model.comment import Comment
db.create_tables([Site, Comment], safe=True) db.create_tables([Comment], safe=True)
# if config.exists(config.DB_BACKUP_JSON_FILE): # if config.exists(config.DB_BACKUP_JSON_FILE):

29
stacosys/core/rss.py
View file

@ -10,44 +10,39 @@ import PyRSS2Gen
import stacosys.conf.config as config import stacosys.conf.config as config
from stacosys.core.templater import Templater, Template from stacosys.core.templater import Templater, Template
from stacosys.model.comment import Comment from stacosys.model.comment import Comment
from stacosys.model.site import Site
class Rss: class Rss:
def __init__(self, lang, rss_file, rss_proto): def __init__(
self, lang, rss_file, rss_proto, site_name, site_url,
):
self._lang = lang self._lang = lang
self._rss_file = rss_file self._rss_file = rss_file
self._rss_proto = rss_proto self._rss_proto = rss_proto
self._site_name = site_name
self._site_url = site_url
current_path = os.path.dirname(__file__) current_path = os.path.dirname(__file__)
template_path = os.path.abspath(os.path.join(current_path, "../templates")) template_path = os.path.abspath(os.path.join(current_path, "../templates"))
self._templater = Templater(template_path) self._templater = Templater(template_path)
def generate_all(self): def generate(self):
for site in Site.select():
self._generate_site(site.token)
def _generate_site(self, token):
site = Site.select().where(Site.token == token).get()
rss_title = self._templater.get_template( rss_title = self._templater.get_template(
self._lang, Template.RSS_TITLE_MESSAGE self._lang, Template.RSS_TITLE_MESSAGE
).render(site=site.name) ).render(site=self._site_name)
md = markdown.Markdown() md = markdown.Markdown()
items = [] items = []
for row in ( for row in (
Comment.select() Comment.select()
.join(Site) .where(Comment.published)
.where(Site.token == token, Comment.published)
.order_by(-Comment.published) .order_by(-Comment.published)
.limit(10) .limit(10)
): ):
item_link = "%s://%s%s" % (self._rss_proto, site.url, row.url) item_link = "%s://%s%s" % (self._rss_proto, self._site_url, row.url)
items.append( items.append(
PyRSS2Gen.RSSItem( PyRSS2Gen.RSSItem(
title="%s - %s://%s%s" title="%s - %s://%s%s"
% (self._rss_proto, row.author_name, site.url, row.url), % (self._rss_proto, row.author_name, self._site_url, row.url),
link=item_link, link=item_link,
description=md.convert(row.content), description=md.convert(row.content),
guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)), guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)),
@ -57,8 +52,8 @@ class Rss:
rss = PyRSS2Gen.RSS2( rss = PyRSS2Gen.RSS2(
title=rss_title, title=rss_title,
link="%s://%s" % (self._rss_proto, site.url), link="%s://%s" % (self._rss_proto, self._site_url),
description='Commentaires du site "%s"' % site.name, description='Commentaires du site "%s"' % self._site_name,
lastBuildDate=datetime.now(), lastBuildDate=datetime.now(),
items=items, items=items,
) )

22
stacosys/interface/api.py
View file

@ -4,9 +4,9 @@
import logging import logging
from flask import abort, jsonify, request from flask import abort, jsonify, request
from stacosys.conf.config import ConfigParameter
from stacosys.interface import app from stacosys.interface import app
from stacosys.model.comment import Comment from stacosys.model.comment import Comment
from stacosys.model.site import Site
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -22,17 +22,15 @@ def query_comments():
comments = [] comments = []
try: try:
token = request.args.get("token", "") token = request.args.get("token", "")
if token != app.config.get(ConfigParameter.SITE_TOKEN):
abort(401)
url = request.args.get("url", "") url = request.args.get("url", "")
logger.info("retrieve comments for url %s" % (url)) logger.info("retrieve comments for url %s" % (url))
for comment in ( for comment in (
Comment.select(Comment) Comment.select(Comment)
.join(Site) .where((Comment.url == url) & (Comment.published.is_null(False)))
.where(
(Comment.url == url)
& (Comment.published.is_null(False))
& (Site.token == token)
)
.order_by(+Comment.published) .order_by(+Comment.published)
): ):
d = { d = {
@ -58,15 +56,13 @@ def query_comments():
def get_comments_count(): def get_comments_count():
try: try:
token = request.args.get("token", "") token = request.args.get("token", "")
if token != app.config.get(ConfigParameter.SITE_TOKEN):
abort(401)
url = request.args.get("url", "") url = request.args.get("url", "")
count = ( count = (
Comment.select(Comment) Comment.select(Comment)
.join(Site) .where((Comment.url == url) & (Comment.published.is_null(False)))
.where(
(Comment.url == url)
& (Comment.published.is_null(False))
& (Site.token == token)
)
.count() .count()
) )
r = jsonify({"count": count}) r = jsonify({"count": count})

9
stacosys/interface/form.py
View file

@ -5,9 +5,9 @@ import logging
from datetime import datetime from datetime import datetime
from flask import abort, redirect, request from flask import abort, redirect, request
from stacosys.conf.config import ConfigParameter
from stacosys.interface import app from stacosys.interface import app
from stacosys.model.comment import Comment from stacosys.model.comment import Comment
from stacosys.model.site import Site
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -21,10 +21,8 @@ def new_form_comment():
# validate token: retrieve site entity # validate token: retrieve site entity
token = data.get("token", "") token = data.get("token", "")
site = Site.select().where(Site.token == token).get() if token != app.config.get(ConfigParameter.SITE_TOKEN):
if site is None: abort(401)
logger.warn("Unknown site %s" % token)
abort(400)
# honeypot for spammers # honeypot for spammers
captcha = data.get("remarque", "") captcha = data.get("remarque", "")
@ -49,7 +47,6 @@ def new_form_comment():
# add a row to Comment table # add a row to Comment table
created = datetime.now().strftime("%Y-%m-%d %H:%M:%S") created = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment = Comment( comment = Comment(
site=site,
url=url, url=url,
author_name=author_name, author_name=author_name,
author_site=author_site, author_site=author_site,

8
stacosys/interface/scheduler.py
View file

@ -11,7 +11,7 @@ class JobConfig(object):
SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 4}} SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 4}}
def __init__(self, imap_polling_seconds, new_comment_polling_seconds, lang, mailer, rss): def __init__(self, imap_polling_seconds, new_comment_polling_seconds, lang, site_name, site_token, site_admin_email, mailer, rss):
self.JOBS = [ self.JOBS = [
{ {
"id": "fetch_mail", "id": "fetch_mail",
@ -23,15 +23,15 @@ class JobConfig(object):
{ {
"id": "submit_new_comment", "id": "submit_new_comment",
"func": "stacosys.core.cron:submit_new_comment", "func": "stacosys.core.cron:submit_new_comment",
"args": [lang, mailer], "args": [lang, site_name, site_token, site_admin_email, mailer],
"trigger": "interval", "trigger": "interval",
"seconds": new_comment_polling_seconds, "seconds": new_comment_polling_seconds,
}, },
] ]
def configure(imap_polling, comment_polling, lang, mailer, rss): def configure(imap_polling, comment_polling, lang, site_name, site_token, site_admin_email, mailer, rss):
app.config.from_object(JobConfig(imap_polling, comment_polling, lang, mailer, rss)) app.config.from_object(JobConfig(imap_polling, comment_polling, lang, site_name, site_token, site_admin_email, mailer, rss))
scheduler = APScheduler() scheduler = APScheduler()
scheduler.init_app(app) scheduler.init_app(app)
scheduler.start() scheduler.start()

3
stacosys/model/comment.py
View file

@ -6,10 +6,10 @@ from peewee import CharField
from peewee import TextField from peewee import TextField
from peewee import DateTimeField from peewee import DateTimeField
from peewee import ForeignKeyField from peewee import ForeignKeyField
from stacosys.model.site import Site
from datetime import datetime from datetime import datetime
from stacosys.core.database import BaseModel from stacosys.core.database import BaseModel
class Comment(BaseModel): class Comment(BaseModel):
url = CharField() url = CharField()
created = DateTimeField() created = DateTimeField()
@ -19,7 +19,6 @@ class Comment(BaseModel):
author_site = CharField(default="") author_site = CharField(default="")
author_gravatar = CharField(default="") author_gravatar = CharField(default="")
content = TextField() content = TextField()
site = ForeignKeyField(Site, related_name="site")
def notify_site_admin(self): def notify_site_admin(self):
self.notified = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.notified = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

13
stacosys/model/site.py
View file

@ -1,13 +0,0 @@
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from peewee import Model
from peewee import CharField
from stacosys.core.database import BaseModel
class Site(BaseModel):
name = CharField(unique=True)
url = CharField()
token = CharField()
admin_email = CharField()

44
tests/test_config.py
View file

@ -2,7 +2,7 @@
# -*- coding: UTF-8 -*- # -*- coding: UTF-8 -*-
import pytest import pytest
from stacosys.conf.config import Config, Parameter from stacosys.conf.config import Config, ConfigParameter
EXPECTED_DB_URL = "sqlite:///db.sqlite" EXPECTED_DB_URL = "sqlite:///db.sqlite"
EXPECTED_HTTP_PORT = 8080 EXPECTED_HTTP_PORT = 8080
@ -13,37 +13,37 @@ EXPECTED_IMAP_LOGIN = "user"
@pytest.fixture @pytest.fixture
def conf(): def conf():
conf = Config() conf = Config()
conf.put(Parameter.DB_URL, EXPECTED_DB_URL) conf.put(ConfigParameter.DB_URL, EXPECTED_DB_URL)
conf.put(Parameter.HTTP_PORT, EXPECTED_HTTP_PORT) conf.put(ConfigParameter.HTTP_PORT, EXPECTED_HTTP_PORT)
conf.put(Parameter.IMAP_PORT, EXPECTED_IMAP_PORT) conf.put(ConfigParameter.IMAP_PORT, EXPECTED_IMAP_PORT)
conf.put(Parameter.SMTP_STARTTLS, "yes") conf.put(ConfigParameter.SMTP_STARTTLS, "yes")
conf.put(Parameter.IMAP_SSL, "false") conf.put(ConfigParameter.IMAP_SSL, "false")
return conf return conf
def test_exists(conf): def test_exists(conf):
assert conf is not None assert conf is not None
assert conf.exists(Parameter.DB_URL) assert conf.exists(ConfigParameter.DB_URL)
assert not conf.exists(Parameter.IMAP_HOST) assert not conf.exists(ConfigParameter.IMAP_HOST)
def test_get(conf): def test_get(conf):
assert conf is not None assert conf is not None
assert conf.get(Parameter.DB_URL) == EXPECTED_DB_URL assert conf.get(ConfigParameter.DB_URL) == EXPECTED_DB_URL
assert conf.get(Parameter.HTTP_PORT) == EXPECTED_HTTP_PORT assert conf.get(ConfigParameter.HTTP_PORT) == EXPECTED_HTTP_PORT
assert conf.get(Parameter.HTTP_HOST) is None assert conf.get(ConfigParameter.HTTP_HOST) is None
assert conf.get(Parameter.HTTP_PORT) == EXPECTED_HTTP_PORT assert conf.get(ConfigParameter.HTTP_PORT) == EXPECTED_HTTP_PORT
assert conf.get(Parameter.IMAP_PORT) == EXPECTED_IMAP_PORT assert conf.get(ConfigParameter.IMAP_PORT) == EXPECTED_IMAP_PORT
assert conf.get_int(Parameter.IMAP_PORT) == int(EXPECTED_IMAP_PORT) assert conf.get_int(ConfigParameter.IMAP_PORT) == int(EXPECTED_IMAP_PORT)
try: try:
conf.get_int(Parameter.HTTP_PORT) conf.get_int(ConfigParameter.HTTP_PORT)
assert False assert False
except: except:
pass pass
assert conf.get_bool(Parameter.SMTP_STARTTLS) assert conf.get_bool(ConfigParameter.SMTP_STARTTLS)
assert not conf.get_bool(Parameter.IMAP_SSL) assert not conf.get_bool(ConfigParameter.IMAP_SSL)
try: try:
conf.get_bool(Parameter.DB_URL) conf.get_bool(ConfigParameter.DB_URL)
assert False assert False
except: except:
pass pass
@ -51,7 +51,7 @@ def test_get(conf):
def test_put(conf): def test_put(conf):
assert conf is not None assert conf is not None
assert not conf.exists(Parameter.IMAP_LOGIN) assert not conf.exists(ConfigParameter.IMAP_LOGIN)
conf.put(Parameter.IMAP_LOGIN, EXPECTED_IMAP_LOGIN) conf.put(ConfigParameter.IMAP_LOGIN, EXPECTED_IMAP_LOGIN)
assert conf.exists(Parameter.IMAP_LOGIN) assert conf.exists(ConfigParameter.IMAP_LOGIN)
assert conf.get(Parameter.IMAP_LOGIN) == EXPECTED_IMAP_LOGIN assert conf.get(ConfigParameter.IMAP_LOGIN) == EXPECTED_IMAP_LOGIN