Add sync support

.gitignore

@@ -3,6 +3,7 @@ media/
 **.sqlite
 database/
 site.conf
+run_8180.py
 
 # automatically generated garbage
 **/__pycache__/

app.py

@@ -8,7 +8,7 @@ Pages are stored in SQLite databases.
 Markdown is used for text formatting.
 
 Application is kept compact, with all its core in a single file.
-Extensions are supported, kept in extensions/ folder.
+Extensions are supported (?), kept in extensions/ folder.
 '''
 
 from flask import (
@@ -101,6 +101,7 @@ class Page(BaseModel):
     touched = DateTimeField(index=True)
     flags = BitField()
     is_redirect = flags.flag(1)
+    is_sync = flags.flag(2)
     @property
     def latest(self):
         if self.revisions:
@@ -125,6 +126,7 @@ class Page(BaseModel):
             title=self.title,
             is_redirect=self.is_redirect,
             touched=self.touched.timestamp(),
+            is_editable=self.is_editable(),
             latest=dict(
                 id=latest.id if latest else None,
                 length=latest.length,
@@ -137,7 +139,7 @@ class Page(BaseModel):
         return PagePropertyDict(self)
     def unlock(self, perm, pp, sec):
         ## XX complete later!
-        policies = self.policies.where(Policy.type << _makelist(perm))
+        policies = self.policies.where(PagePolicy.type << _makelist(perm))
         if not policies.exists():
             return True
         for policy in policies:
@@ -145,10 +147,12 @@ class Page(BaseModel):
                 return True
         return False
     def is_locked(self, perm):
-        policies = self.policies.where(Policy.type << _makelist(perm))
+        policies = self.policies.where(PagePolicy.type << _makelist(perm))
         return policies.exists()
     def is_classified(self):
         return self.is_locked(POLICY_CLASSIFY)
+    def is_editable(self):
+        return not self.is_locked(POLICY_EDIT)
             
 
 class PageText(BaseModel):
@@ -570,8 +574,8 @@ def create():
                 touched=datetime.datetime.now(),
             )
             p.change_tags(p_tags)
-        except IntegrityError:
-            flash('An error occurred while saving this revision.')
+        except IntegrityError as e:
+            flash('An error occurred while saving this revision: {e}'.format(e=e))
             return savepoint(request.form)
         pr = PageRevision.create(
             page=p,
@@ -620,6 +624,37 @@ def edit(id):
         return redirect(p.get_url())
     return render_template('edit.html', pl_url=p.url, pl_title=p.title, pl_text=p.latest.text, pl_tags=','.join(x.name for x in p.tags))
 
+@app.route("/__sync_start")
+def __sync_start():
+    if _getconf("sync", "master", "this") == "this":
+        abort(403)
+    from app_sync import main
+    main()
+    flash("Successfully synced messages.")
+    return redirect("/")
+
+@app.route('/_jsoninfo/<int:id>', methods=['GET', 'POST'])
+def page_jsoninfo(id):
+    try:
+        p = Page[id]
+    except Page.DoesNotExist:
+        return jsonify({'status':'fail'}), 404
+    j = p.js_info()
+    j["status"] = "ok"
+    if request.method == "POST":
+        j["text"] = p.latest.text
+    return jsonify(j)
+
+@app.route("/_jsoninfo/changed/<float:ts>")
+def jsoninfo_changed(ts):
+    tse = str(datetime.datetime.fromtimestamp(ts).isoformat(" "))
+    ps = Page.select().where(Page.touched >= tse)
+    return jsonify({
+        "ids": [i.id for i in ps],
+        "status": "ok"
+    })
+
+    
 @app.route('/p/<int:id>/')
 def view_unnamed(id):
     try:

app_sync.py

@@ -0,0 +1,133 @@
+"""
+Helper module for sync.
+
+(c) 2021 Sakuragasaki46.
+"""
+
+import datetime, time
+import requests
+import sys, os
+from configparser import ConfigParser
+from app import Page, PageRevision, PageText
+from peewee import IntegrityError
+from functools import lru_cache
+
+## CONSTANTS ##
+
+APP_BASE_DIR = os.path.dirname(__file__)
+
+UPLOAD_DIR = APP_BASE_DIR + '/media'
+DATABASE_DIR = APP_BASE_DIR + "/database"
+
+#### GENERAL CONFIG ####
+
+DEFAULT_CONF = {
+    ('site', 'title'):          'Salvi',
+    ('config', 'media_dir'):    APP_BASE_DIR + '/media',
+    ('config', 'database_dir'): APP_BASE_DIR + "/database",
+}
+
+_cfp = ConfigParser()
+if _cfp.read([APP_BASE_DIR + '/site.conf']):
+    @lru_cache(maxsize=50)
+    def _getconf(k1, k2, fallback=None):
+        if fallback is None:
+            fallback = DEFAULT_CONF.get((k1, k2))
+        v = _cfp.get(k1, k2, fallback=fallback)
+        return v
+else:
+    def _getconf(k1, k2, fallback=None):
+        if fallback is None:
+            fallback = DEFAULT_CONF.get((k1, k2))
+        return fallback
+
+#### misc. helpers ####
+
+def _makelist(l):
+    if isinstance(l, (str, bytes, bytearray)):
+        return [l]
+    elif hasattr(l, '__iter__'):
+        return list(l)
+    elif l:
+        return [l]
+    else:
+        return []
+
+#### REQUESTS ####
+
+def fetch_updated_ids(baseurl):
+    try:
+        with open(_getconf("config", "database_dir") + "/latest_sync") as f:
+            last_sync = float(f.read().rstrip("\n"))
+    except (OSError, ValueError):
+        last_sync = 946681200.0  # Jan 1, 2000
+    r = requests.get(baseurl + "/_jsoninfo/changed/{ts}".format(ts=last_sync))
+    if r.status_code >= 400:
+        raise RuntimeError("sync unavailable")
+    return r.json()["ids"]
+
+def update_page(p, pageinfo):
+    p.touched = datetime.datetime.fromtimestamp(pageinfo["touched"])
+    p.url = pageinfo["url"]
+    p.title = pageinfo["title"]
+    p.save()
+    p.change_tags(pageinfo["tags"])
+    assert len(pageinfo["text"]) == pageinfo["latest"]["length"]
+    pr = PageRevision.create(
+        page=p,
+        user_id=0,
+        comment='',
+        textref=PageText.create_content(pageinfo['text']),
+        pub_date=datetime.datetime.fromtimestamp(pageinfo["latest"]["pub_date"]),
+        length=pageinfo["latest"]["length"]
+    )
+
+#### MAIN ####
+
+def main():
+    baseurl = _getconf("sync", "master", "this")
+    if baseurl == "this":
+        print("unsyncable: master", file=sys.stderr)
+        return
+    if not baseurl.startswith(("http:", "https:")):
+        print("unsyncable: invalid url", repr(baseurl), file=sys.stderr)
+        return
+    passed, failed = 0, 0
+    for i in fetch_updated_ids(baseurl):
+        pageinfo_r = requests.post(baseurl + "/_jsoninfo/{i}".format(i=i))
+        if pageinfo_r.status_code >= 400:
+            print("\x1b[31mSkipping {i}: HTTP {s}\x1b[0m".format(i=i, s=pageinfo_r.status_code))
+            failed += 1
+            continue
+        pageinfo = pageinfo_r.json()
+        try:
+            p = Page[i]
+        except Page.DoesNotExist:
+            try:
+                p = Page.create(
+                    id=i,
+                    url=pageinfo['url'],
+                    title=pageinfo['title'],
+                    is_redirect=pageinfo['is_redirect'],
+                    touched=datetime.datetime.fromtimestamp(pageinfo["touched"]),
+                    is_sync = True
+                )
+                update_page(p, pageinfo)
+            except IntegrityError:
+                print("\x1b[31mSkipping {i}: Integrity error\x1b[0m".format(i=i))
+                failed += 1
+                continue
+        else:
+            if pageinfo["touched"] > p.touched:
+                update_page(p, pageinfo)
+        passed += 1
+    with open(DATABASE_DIR + "/last_sync", "w") as fw:
+        fw.write(str(time.time()))
+    if passed > 0 and failed == 0:
+        print("\x1b[32mSuccessfully updated {p} pages :)\x1b[0m".format(p=passed))
+    else:
+        print("\x1b[33m{p} pages successfully updated, {f} errors.\x1b[0m".format(p=passed, f=failed))
+    
+
+if __name__ == "__main__":
+    main()

templates/base.html

@@ -7,6 +7,7 @@
     <link rel="stylesheet" href="/static/style.css">
     <!-- material icons -->
     <link rel="stylesheet" href="https://cdn.sakuragasaki46.local/common/material-icons.css">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
     {% block json_info %}{% endblock %}
   </head>
   <body{% if request.cookies.get('dark') == '1' %} class="dark"{% endif %}>