""" Helper module for sync. Remember to set sync:master variable in site.conf! (c) 2021 Sakuragasaki46. """ import datetime, time import requests import sys, os from configparser import ConfigParser from app import Page, PageRevision, PageText from peewee import IntegrityError from functools import lru_cache ## CONSTANTS ## APP_BASE_DIR = os.path.dirname(__file__) UPLOAD_DIR = APP_BASE_DIR + '/media' DATABASE_DIR = APP_BASE_DIR + "/database" #### GENERAL CONFIG #### DEFAULT_CONF = { ('site', 'title'): 'Salvi', ('config', 'media_dir'): APP_BASE_DIR + '/media', ('config', 'database_dir'): APP_BASE_DIR + "/database", } _cfp = ConfigParser() if _cfp.read([APP_BASE_DIR + '/site.conf']): @lru_cache(maxsize=50) def _getconf(k1, k2, fallback=None): if fallback is None: fallback = DEFAULT_CONF.get((k1, k2)) v = _cfp.get(k1, k2, fallback=fallback) return v else: def _getconf(k1, k2, fallback=None): if fallback is None: fallback = DEFAULT_CONF.get((k1, k2)) return fallback #### misc. helpers #### def _makelist(l): if isinstance(l, (str, bytes, bytearray)): return [l] elif hasattr(l, '__iter__'): return list(l) elif l: return [l] else: return [] #### REQUESTS #### def fetch_updated_ids(baseurl): try: with open(_getconf("config", "database_dir") + "/latest_sync") as f: last_sync = float(f.read().rstrip("\n")) except (OSError, ValueError): last_sync = 946681200.0 # Jan 1, 2000 r = requests.get(baseurl + "/_jsoninfo/changed/{ts}".format(ts=last_sync)) if r.status_code >= 400: raise RuntimeError("sync unavailable") return r.json()["ids"] def update_contacts(baseurl): from extensions.contactnova import Contact try: with open(_getconf("config", "database_dir") + "/latest_sync") as f: last_sync = float(f.read().rstrip("\n")) except (OSError, ValueError): last_sync = 946681200.0 # Jan 1, 2000 r = requests.get(baseurl + "/kt/_jsoninfo/{ts}".format(ts=last_sync)) if r.status_code >= 400: raise RuntimeError("sync unavailable") # update contacts updated = 0 for pinfo in r.json()['data']: p = Contact.get_or_none(Contact.code == pinfo['code']) if p is None: p = Contact.create( code = pinfo['code'], display_name = pinfo['display_name'], issues = pinfo['issues'], status = pinfo['status'], description = pinfo['description'], due = datetime.date.fromtimestamp(pinfo['due']) ) else: p.display_name = pinfo['display_name'] p.issues = pinfo['issues'] p.status = pinfo['status'] p.description = pinfo['description'] p.due = datetime.date.fromtimestamp(pinfo['due']) p.touched = datetime.datetime.now() p.save() updated += 1 print('\x1b[32m{0} contacts updated :)\x1b[0m'.format(updated)) def update_page(p, pageinfo): p.touched = datetime.datetime.fromtimestamp(pageinfo["touched"]) p.url = pageinfo["url"] p.title = pageinfo["title"] p.save() p.change_tags(pageinfo["tags"]) assert len(pageinfo["text"]) == pageinfo["latest"]["length"] pr = PageRevision.create( page=p, user_id=0, comment='', textref=PageText.create_content(pageinfo['text']), pub_date=datetime.datetime.fromtimestamp(pageinfo["latest"]["pub_date"]), length=pageinfo["latest"]["length"] ) #### MAIN #### def main(): baseurl = _getconf("sync", "master", "this") if baseurl == "this": print("unsyncable: master", file=sys.stderr) return if not baseurl.startswith(("http:", "https:")): print("unsyncable: invalid url", repr(baseurl), file=sys.stderr) return passed, failed = 0, 0 for i in fetch_updated_ids(baseurl): pageinfo_r = requests.post(baseurl + "/_jsoninfo/{i}".format(i=i)) if pageinfo_r.status_code >= 400: print("\x1b[31mSkipping {i}: HTTP {s}\x1b[0m".format(i=i, s=pageinfo_r.status_code)) failed += 1 continue pageinfo = pageinfo_r.json() try: p = Page[i] except Page.DoesNotExist: try: p = Page.create( id=i, url=pageinfo['url'], title=pageinfo['title'], is_redirect=pageinfo['is_redirect'], touched=datetime.datetime.fromtimestamp(pageinfo["touched"]), is_sync = True ) update_page(p, pageinfo) except IntegrityError: print("\x1b[31mSkipping {i}: Integrity error\x1b[0m".format(i=i)) failed += 1 continue else: if pageinfo["touched"] > p.touched.timestamp(): update_page(p, pageinfo) passed += 1 try: if _getconf("sync", "contacts", None) is not None: update_contacts(baseurl) except Exception as e: print("\x1b[33mContacts not updated - {e} :(\x1b[0m".format(e=e)) with open(DATABASE_DIR + "/last_sync", "w") as fw: fw.write(str(time.time())) if passed > 0 and failed == 0: print("\x1b[32mSuccessfully updated {p} pages :)\x1b[0m".format(p=passed)) else: print("\x1b[33m{p} pages successfully updated, {f} errors.\x1b[0m".format(p=passed, f=failed)) if __name__ == "__main__": main()