diff --git a/.gitignore b/.gitignore index ad11ee5..8a5dbec 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ media/ **.sqlite database/ site.conf +run_8180.py # automatically generated garbage **/__pycache__/ diff --git a/app.py b/app.py index 9d794ed..e7c947c 100644 --- a/app.py +++ b/app.py @@ -8,7 +8,7 @@ Pages are stored in SQLite databases. Markdown is used for text formatting. Application is kept compact, with all its core in a single file. -Extensions are supported, kept in extensions/ folder. +Extensions are supported (?), kept in extensions/ folder. ''' from flask import ( @@ -101,6 +101,7 @@ class Page(BaseModel): touched = DateTimeField(index=True) flags = BitField() is_redirect = flags.flag(1) + is_sync = flags.flag(2) @property def latest(self): if self.revisions: @@ -125,6 +126,7 @@ class Page(BaseModel): title=self.title, is_redirect=self.is_redirect, touched=self.touched.timestamp(), + is_editable=self.is_editable(), latest=dict( id=latest.id if latest else None, length=latest.length, @@ -137,7 +139,7 @@ class Page(BaseModel): return PagePropertyDict(self) def unlock(self, perm, pp, sec): ## XX complete later! - policies = self.policies.where(Policy.type << _makelist(perm)) + policies = self.policies.where(PagePolicy.type << _makelist(perm)) if not policies.exists(): return True for policy in policies: @@ -145,10 +147,12 @@ class Page(BaseModel): return True return False def is_locked(self, perm): - policies = self.policies.where(Policy.type << _makelist(perm)) + policies = self.policies.where(PagePolicy.type << _makelist(perm)) return policies.exists() def is_classified(self): return self.is_locked(POLICY_CLASSIFY) + def is_editable(self): + return not self.is_locked(POLICY_EDIT) class PageText(BaseModel): @@ -570,8 +574,8 @@ def create(): touched=datetime.datetime.now(), ) p.change_tags(p_tags) - except IntegrityError: - flash('An error occurred while saving this revision.') + except IntegrityError as e: + flash('An error occurred while saving this revision: {e}'.format(e=e)) return savepoint(request.form) pr = PageRevision.create( page=p, @@ -620,6 +624,37 @@ def edit(id): return redirect(p.get_url()) return render_template('edit.html', pl_url=p.url, pl_title=p.title, pl_text=p.latest.text, pl_tags=','.join(x.name for x in p.tags)) +@app.route("/__sync_start") +def __sync_start(): + if _getconf("sync", "master", "this") == "this": + abort(403) + from app_sync import main + main() + flash("Successfully synced messages.") + return redirect("/") + +@app.route('/_jsoninfo/', methods=['GET', 'POST']) +def page_jsoninfo(id): + try: + p = Page[id] + except Page.DoesNotExist: + return jsonify({'status':'fail'}), 404 + j = p.js_info() + j["status"] = "ok" + if request.method == "POST": + j["text"] = p.latest.text + return jsonify(j) + +@app.route("/_jsoninfo/changed/") +def jsoninfo_changed(ts): + tse = str(datetime.datetime.fromtimestamp(ts).isoformat(" ")) + ps = Page.select().where(Page.touched >= tse) + return jsonify({ + "ids": [i.id for i in ps], + "status": "ok" + }) + + @app.route('/p//') def view_unnamed(id): try: diff --git a/app_sync.py b/app_sync.py new file mode 100644 index 0000000..9dbc123 --- /dev/null +++ b/app_sync.py @@ -0,0 +1,133 @@ +""" +Helper module for sync. + +(c) 2021 Sakuragasaki46. +""" + +import datetime, time +import requests +import sys, os +from configparser import ConfigParser +from app import Page, PageRevision, PageText +from peewee import IntegrityError +from functools import lru_cache + +## CONSTANTS ## + +APP_BASE_DIR = os.path.dirname(__file__) + +UPLOAD_DIR = APP_BASE_DIR + '/media' +DATABASE_DIR = APP_BASE_DIR + "/database" + +#### GENERAL CONFIG #### + +DEFAULT_CONF = { + ('site', 'title'): 'Salvi', + ('config', 'media_dir'): APP_BASE_DIR + '/media', + ('config', 'database_dir'): APP_BASE_DIR + "/database", +} + +_cfp = ConfigParser() +if _cfp.read([APP_BASE_DIR + '/site.conf']): + @lru_cache(maxsize=50) + def _getconf(k1, k2, fallback=None): + if fallback is None: + fallback = DEFAULT_CONF.get((k1, k2)) + v = _cfp.get(k1, k2, fallback=fallback) + return v +else: + def _getconf(k1, k2, fallback=None): + if fallback is None: + fallback = DEFAULT_CONF.get((k1, k2)) + return fallback + +#### misc. helpers #### + +def _makelist(l): + if isinstance(l, (str, bytes, bytearray)): + return [l] + elif hasattr(l, '__iter__'): + return list(l) + elif l: + return [l] + else: + return [] + +#### REQUESTS #### + +def fetch_updated_ids(baseurl): + try: + with open(_getconf("config", "database_dir") + "/latest_sync") as f: + last_sync = float(f.read().rstrip("\n")) + except (OSError, ValueError): + last_sync = 946681200.0 # Jan 1, 2000 + r = requests.get(baseurl + "/_jsoninfo/changed/{ts}".format(ts=last_sync)) + if r.status_code >= 400: + raise RuntimeError("sync unavailable") + return r.json()["ids"] + +def update_page(p, pageinfo): + p.touched = datetime.datetime.fromtimestamp(pageinfo["touched"]) + p.url = pageinfo["url"] + p.title = pageinfo["title"] + p.save() + p.change_tags(pageinfo["tags"]) + assert len(pageinfo["text"]) == pageinfo["latest"]["length"] + pr = PageRevision.create( + page=p, + user_id=0, + comment='', + textref=PageText.create_content(pageinfo['text']), + pub_date=datetime.datetime.fromtimestamp(pageinfo["latest"]["pub_date"]), + length=pageinfo["latest"]["length"] + ) + +#### MAIN #### + +def main(): + baseurl = _getconf("sync", "master", "this") + if baseurl == "this": + print("unsyncable: master", file=sys.stderr) + return + if not baseurl.startswith(("http:", "https:")): + print("unsyncable: invalid url", repr(baseurl), file=sys.stderr) + return + passed, failed = 0, 0 + for i in fetch_updated_ids(baseurl): + pageinfo_r = requests.post(baseurl + "/_jsoninfo/{i}".format(i=i)) + if pageinfo_r.status_code >= 400: + print("\x1b[31mSkipping {i}: HTTP {s}\x1b[0m".format(i=i, s=pageinfo_r.status_code)) + failed += 1 + continue + pageinfo = pageinfo_r.json() + try: + p = Page[i] + except Page.DoesNotExist: + try: + p = Page.create( + id=i, + url=pageinfo['url'], + title=pageinfo['title'], + is_redirect=pageinfo['is_redirect'], + touched=datetime.datetime.fromtimestamp(pageinfo["touched"]), + is_sync = True + ) + update_page(p, pageinfo) + except IntegrityError: + print("\x1b[31mSkipping {i}: Integrity error\x1b[0m".format(i=i)) + failed += 1 + continue + else: + if pageinfo["touched"] > p.touched: + update_page(p, pageinfo) + passed += 1 + with open(DATABASE_DIR + "/last_sync", "w") as fw: + fw.write(str(time.time())) + if passed > 0 and failed == 0: + print("\x1b[32mSuccessfully updated {p} pages :)\x1b[0m".format(p=passed)) + else: + print("\x1b[33m{p} pages successfully updated, {f} errors.\x1b[0m".format(p=passed, f=failed)) + + +if __name__ == "__main__": + main() diff --git a/templates/base.html b/templates/base.html index 3f2b238..a9eea53 100644 --- a/templates/base.html +++ b/templates/base.html @@ -7,6 +7,7 @@ + {% block json_info %}{% endblock %}