Conversion tool is near from complete. Use Peewee as ORM

app/models/comment.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+from peewee import Model
+from peewee import CharField
+from peewee import DateTimeField
+from peewee import IntegerField
+from peewee import ForeignKeyField
+from app.models.site import Site
+from app.services.database import get_db
+
+
+class Comment(Model):
+    url = CharField()
+    date = DateTimeField()
+    rel_index = IntegerField()
+    author_name = CharField()
+    author_email = CharField(default='')
+    author_site = CharField()
+    content = CharField()
+    site = ForeignKeyField(Site, related_name='site')
+
+    class Meta:
+        database = get_db()

app/models/site.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+from peewee import Model
+from peewee import CharField
+from app.services.database import get_db
+
+
+class Site(Model):
+    name = CharField(unique=True)
+    url = CharField()
+    token = CharField()
+
+    class Meta:
+        database = get_db()

app/services/database.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+import hashlib
+import config
+import functools
+from config import DB_URL
+from playhouse.db_url import connect
+
+
+def get_db():
+    return connect(DB_URL)
+
+
+def provide_db(func):
+
+    @functools.wraps(func)
+    def new_function(*args, **kwargs):
+        return func(get_db(), *args, **kwargs)
+
+    return new_function
+
+
+def hash(value):
+    string = '%s%s' % (value, config.SALT)
+    dk = hashlib.sha256(string.encode())
+    return dk.hexdigest()
+
+
+@provide_db
+def setup(db):
+    from app.models.user import User
+    db.create_tables([User], safe=True)
+
+    # create admin user if user table is empty
+    if User.select().count() == 0:
+        admin_user = User(username='admin', password=hash('admin'),
+                          displayname='Admin')
+        admin_user.save()

config.py

@@ -0,0 +1,3 @@
+# Configuration file
+
+DB_URL = "mysql://stacosys_user:stacosys_password@localhost:3306/stacosys"

requirements.txt

@@ -1,6 +1,8 @@
+clize==2.4
 Flask==0.10.1
 itsdangerous==0.24
 Jinja2==2.7.3
 MarkupSafe==0.23
-SQLAlchemy==1.0.2
+peewee==2.6.0
+PyMySQL==0.6.6
 Werkzeug==0.10.4

tools/config.json

@@ -1,3 +0,0 @@
-{
-    "comments": "/home/yannic/work/coding/blog/blogduyax/comments"
-}

tools/pecosys2stacosys.py

@@ -1,12 +1,24 @@
 #!/usr/bin/env python
 # -*- coding:utf-8 -*-
 
+import sys
 import os
 import re
-import json
 import logging
 from clize import clize, run
 
+# add necessary directories to PATH
+current_path = os.path.realpath('.')
+parent_path = os.path.abspath(os.path.join(current_path, '..'))
+paths = [current_path, parent_path]
+for path in paths:
+    if path not in sys.path:
+        sys.path.insert(0, path)
+
+# import database models 
+from app.services.database import provide_db
+from app.models.site import Site
+from app.models.comment import Comment
 
 # configure logging
 level = logging.DEBUG
@@ -22,9 +34,10 @@ logger.addHandler(ch)
 regex = re.compile(r"(\w+):\s*(.*)")
 
 
-def convert_comment(config, filename):
+def convert_comment(db, site, filename):
     logger.info('convert %s' % filename)
     d = {}
+    content = ''
     with open(filename) as f:
         for line in f:
             match = regex.match(line)
@@ -32,32 +45,60 @@ def convert_comment(config, filename):
                 d[match.group(1)] = match.group(2)
             else:
                 break
+        is_header = True
+        for line in f:
+            if is_header:
+                if line.strip():
+                    is_header = False
+                else:
+                    continue
+            content = content + line
     logger.debug(d)
+    logger.debug(content)
+
+    # create DB record
+    comment = Comment(site=site, author_name=d['author'], content=content)
+    if 'email' in d:
+        comment.author_email = d['email']
+    if 'site' in d:
+        comment.author_site = d['site']
+    if 'url' in d:
+        comment.url = d['url']
+    # else:
+    #    comment.url = d['article']
+    if 'date' in d:
+        comment.date = d['date']
+    comment.save()
 
 
-def convert(config):
+@provide_db
-    comment_dir = config['comments']
+def convert(db, site_name, url, comment_dir):
+
+    # create DB tables if needed
+    db.create_tables([Site, Comment], safe=True)
+
+    # delete site record
+    try:
+        site = Site.select().where(Site.name == site_name).get()
+        site.delete_instance(recursive=True)
+    except Site.DoesNotExist:
+        pass
+
+    site = Site.create(name=site_name, url=url, token='')
+
     logger.info('Comment directory %s' % comment_dir)
     for dirpath, dirs, files in os.walk(comment_dir):
         for filename in files:
             if filename.endswith(('.md',)):
                 comment_file = '/'.join([dirpath, filename])
-                convert_comment(config, comment_file)
+                convert_comment(db, site, comment_file)
             else:
                 logger.debug('ignore file %s' % filename)
 
 
-def load_config(config_pathname):
-    logger.info("Load config from %s" % config_pathname)
-    with open(config_pathname, 'rt') as config_file:
-        config = json.loads(config_file.read())
-        return config
-
-
 @clize
-def pecosys2stacosys(config_pathname):
+def pecosys2stacosys(site, url, comment_dir):
-    config = load_config(config_pathname)
+    convert(site, url, comment_dir)
-    convert(config)
 
 
 if __name__ == '__main__':