diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..a283d9d --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,79 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from salvi.models import Base +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/ae0587e14725_.py b/alembic/versions/ae0587e14725_.py new file mode 100644 index 0000000..766b472 --- /dev/null +++ b/alembic/versions/ae0587e14725_.py @@ -0,0 +1,86 @@ +"""empty message + +Sorry, due to unattended changes, upgrade from 1.0.0 is not possible. Sorry. + +Revision ID: ae0587e14725 +Revises: +Create Date: 2025-10-04 09:43:41.158057 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision: str = 'ae0587e14725' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + #op.drop_index('pagepolicykey_passphrase_sec_code', table_name='pagepolicykey') + #op.drop_index('page_owner', table_name='page') + op.create_index(op.f('ix_page_calendar'), 'page', ['calendar'], unique=False) + op.create_index('page_calendar', 'page', ['calendar'], unique=False) + op.drop_index('user_id', table_name='usergroupmembership') + op.drop_index('user_id_2', table_name='usergroupmembership') + #op.drop_index('usergroupmembership_group_id', table_name='usergroupmembership') + #op.drop_index('usergroupmembership_user_id', table_name='usergroupmembership') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_index('usergroupmembership_user_id', 'usergroupmembership', ['user_id'], unique=False) + op.create_index('usergroupmembership_group_id', 'usergroupmembership', ['group_id'], unique=False) + op.create_index('user_id_2', 'usergroupmembership', ['user_id', 'group_id'], unique=True) + op.create_index('user_id', 'usergroupmembership', ['user_id', 'group_id'], unique=True) + op.drop_index('page_calendar', table_name='page') + op.drop_index(op.f('ix_page_calendar'), table_name='page') + op.create_index('page_owner', 'page', ['owner_id'], unique=False) + op.create_table('upload', + sa.Column('id', mysql.INTEGER(display_width=11), autoincrement=True, nullable=False), + sa.Column('name', mysql.VARCHAR(length=256), nullable=False), + sa.Column('url_name', mysql.VARCHAR(length=256), nullable=True), + sa.Column('filetype', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False), + sa.Column('filesize', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False), + sa.Column('upload_date', mysql.DATETIME(), nullable=False), + sa.Column('md5', mysql.VARCHAR(length=32), nullable=False), + sa.PrimaryKeyConstraint('id'), + mariadb_collate='utf8mb4_general_ci', + mariadb_default_charset='utf8mb4', + mariadb_engine='InnoDB' + ) + op.create_index('upload_upload_date', 'upload', ['upload_date'], unique=False) + op.create_index('upload_md5', 'upload', ['md5'], unique=False) + op.create_table('pagepolicy', + sa.Column('id', mysql.INTEGER(display_width=11), autoincrement=True, nullable=False), + sa.Column('page_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True), + sa.Column('type', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False), + sa.Column('key_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False), + sa.Column('sitewide', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['key_id'], ['pagepolicykey.id'], name='pagepolicy_FK_0_0'), + sa.ForeignKeyConstraint(['page_id'], ['page.id'], name='pagepolicy_FK_1_0'), + sa.PrimaryKeyConstraint('id'), + mariadb_collate='utf8mb4_general_ci', + mariadb_default_charset='utf8mb4', + mariadb_engine='InnoDB' + ) + op.create_index('pagepolicy_page_id_key_id', 'pagepolicy', ['page_id', 'key_id'], unique=True) + op.create_index('pagepolicy_page_id', 'pagepolicy', ['page_id'], unique=False) + op.create_index('pagepolicy_key_id', 'pagepolicy', ['key_id'], unique=False) + op.create_table('pagepolicykey', + sa.Column('id', mysql.INTEGER(display_width=11), autoincrement=True, nullable=False), + sa.Column('passphrase', mysql.VARCHAR(length=255), nullable=False), + sa.Column('sec_code', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id'), + mariadb_collate='utf8mb4_general_ci', + mariadb_default_charset='utf8mb4', + mariadb_engine='InnoDB' + ) + op.create_index('pagepolicykey_passphrase_sec_code', 'pagepolicykey', ['passphrase', 'sec_code'], unique=True) + # ### end Alembic commands ### diff --git a/pyproject.toml b/pyproject.toml index ffd0921..5dca1b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "Flask-WTF", "python-dotenv>=1.0.0", "pymysql", - "sakuragasaki46-suou>=0.6.0" + "sakuragasaki46-suou>=0.7.0" ] requires-python = ">=3.10" classifiers = [ diff --git a/salvi/__init__.py b/salvi/__init__.py index f6c016a..4afe92b 100644 --- a/salvi/__init__.py +++ b/salvi/__init__.py @@ -8,7 +8,7 @@ Pages are stored in SQLite/MySQL databases. Markdown is used for text formatting. ''' -__version__ = '1.1.0-dev36' +__version__ = '1.1.0-dev40' from flask import ( Flask, g, make_response, redirect, diff --git a/salvi/models.py b/salvi/models.py index 67f6260..a373cf0 100644 --- a/salvi/models.py +++ b/salvi/models.py @@ -6,22 +6,20 @@ from __future__ import annotations from functools import partial, wraps from getpass import getpass -import os import re import sys from typing import Any, Callable, Iterable, List, Mapping -import warnings +import datetime +import gzip -# sqlalchemy imports from flask import abort, flash from flask_login import AnonymousUserMixin, current_user, login_required from flask_sqlalchemy import SQLAlchemy from markupsafe import Markup -from sqlalchemy.orm import Mapped, Relationship, declarative_base, deferred, relationship -from sqlalchemy import Column, Integer, String, DateTime, BigInteger, ForeignKey, UniqueConstraint, create_engine, Index, BLOB as Blob, delete, func, insert, or_, select, update -import datetime -import os -import gzip + +# sqlalchemy imports +from sqlalchemy.orm import Mapped, Relationship, declarative_base, mapped_column, relationship +from sqlalchemy import Index, Integer, SmallInteger, String, DateTime, BigInteger, ForeignKey, UniqueConstraint, BLOB as Blob, delete, func, insert, or_, select, text, update from suou.functools import deprecated, deprecated_alias, not_implemented from werkzeug.security import generate_password_hash @@ -34,7 +32,7 @@ from .i18n import human_elapsed_time current_user: User # Helper for interactive session management -from suou.sqlalchemy import create_session, declarative_base, BitSelector +from suou.sqlalchemy import bool_column, bound_fk, create_session, declarative_base, BitSelector, unbound_fk CSI = create_session_interactively = partial(create_session, app_config.database_url) @@ -45,21 +43,23 @@ db = SQLAlchemy(model_class=Base) class User(db.Model): __tablename__ = 'user' - id = Column(Integer, primary_key=True) - username = Column(String(32), unique=True) - email = Column(String(256), nullable=True) - password = Column(String(255)) - # will change to server_default in 1.1 - join_date = Column(DateTime, default=datetime.datetime.now) - karma = Column(Integer, default=1) - privileges = Column(BigInteger, default=0) - is_admin = BitSelector(privileges, 1) - restrictions = Column(BigInteger, default=0) - is_disabled = BitSelector(restrictions, 1) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + username: Mapped[str] = mapped_column(String(32), unique=True) + email: Mapped[str | None] = mapped_column(String(256), nullable=True) + password: Mapped[str] = mapped_column(String(255)) + join_date: Mapped[int] = mapped_column(DateTime, server_default=func.current_timestamp()) + karma: Mapped[int] = mapped_column(Integer, server_default=text('1')) + is_admin: Mapped[bool] = bool_column() + is_disabled: Mapped[int] = bool_column() + color_theme: Mapped[int] = mapped_column(SmallInteger, server_default=text('0')) owned_pages: Relationship[List[Page]] = relationship('Page', back_populates='owner') group_memberships: Relationship[UserGroupMembership] = relationship('UserGroupMembership', back_populates='user') - contributions = relationship("PageRevision", back_populates='user') + contributions: Relationship[PageRevision] = relationship("PageRevision", back_populates='user') + + __table_args__ = ( + UniqueConstraint(username, name='user_username'), + ) # helpers for flask_login @property @@ -119,14 +119,18 @@ PERM_LOCK = PERM_READ class UserGroup(db.Model): __tablename__ = 'usergroup' - id = Column(Integer, primary_key=True) - name = Column(String(32), unique=True) - permissions = Column(BigInteger, default=0) - can_read = BitSelector(permissions, PERM_READ) - can_edit = BitSelector(permissions, PERM_EDIT) - can_create = BitSelector(permissions, PERM_CREATE) - can_set_url = BitSelector(permissions, PERM_SET_URL) - can_set_tags = BitSelector(permissions, PERM_SET_TAGS) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + name: Mapped[str] = mapped_column(String(32), unique=True) + permissions: Mapped[int] = mapped_column(BigInteger, server_default=text('0')) + can_read: Mapped[bool] = BitSelector(permissions, PERM_READ) + can_edit: Mapped[bool] = BitSelector(permissions, PERM_EDIT) + can_create: Mapped[bool] = BitSelector(permissions, PERM_CREATE) + can_set_url: Mapped[bool] = BitSelector(permissions, PERM_SET_URL) + can_set_tags: Mapped[bool] = BitSelector(permissions, PERM_SET_TAGS) + + __table_args__ = ( + UniqueConstraint(name, name='usergroup_name'), + ) user_memberships: Relationship[UserGroupMembership] = relationship('UserGroupMembership', back_populates='group') page_permissions: Relationship[PagePermission] = relationship('PagePermission', back_populates = 'group') @@ -147,14 +151,15 @@ class UserGroup(db.Model): class UserGroupMembership(db.Model): __tablename__ = 'usergroupmembership' - __table_args__ = ( - UniqueConstraint('user_id', 'group_id'), - ) - id = Column(Integer, primary_key = True) - user_id = Column(Integer, ForeignKey('user.id')) - group_id = Column(Integer, ForeignKey('usergroup.id')) - since = Column(DateTime, default=datetime.datetime.now) + id: Mapped[int] = mapped_column(Integer, primary_key = True) + user_id: Mapped[int] = mapped_column(Integer, ForeignKey('user.id')) + group_id: Mapped[int] = mapped_column(Integer, ForeignKey('usergroup.id')) + since: Mapped[datetime.datetime] = mapped_column(DateTime, default=datetime.datetime.now) + + __table_args__ = ( + UniqueConstraint(user_id, group_id, name="usergroupmembership_user_id_group_id"), + ) user: Relationship[User] = relationship("User", back_populates='group_memberships') group: Relationship[UserGroup] = relationship("UserGroup", back_populates='user_memberships') @@ -163,19 +168,27 @@ class UserGroupMembership(db.Model): class Page(db.Model): __tablename__ = 'page' - id = Column(Integer, primary_key=True) - url = Column(String(64), unique=True, nullable=True) - title = Column(String(256), index=True) - touched = Column(DateTime, index=True) - calendar = Column(DateTime, index=True, nullable=True) - owner_id = Column(Integer, ForeignKey('user.id'), nullable=True) - flags = Column(BigInteger, default=0) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + # v--- url will be moved to a separate table in 1.2 (?) + url: Mapped[str | None] = mapped_column(String(64), unique=True) + title: Mapped[str] = mapped_column(String(256)) + touched: Mapped[datetime.datetime] = mapped_column(DateTime) + calendar: Mapped[datetime.datetime | None] = mapped_column(DateTime, index=True) + owner_id: Mapped[int | None] = mapped_column(Integer, ForeignKey('user.id')) + flags: Mapped[int] = mapped_column(BigInteger, server_default=text('0')) is_redirect: Mapped[bool] = BitSelector(flags, 1) is_sync: Mapped[bool] = BitSelector(flags, 2) is_math_enabled: Mapped[bool] = BitSelector(flags, 4) is_locked: Mapped[bool] = BitSelector(flags, 8) is_cw: Mapped[bool] = BitSelector(flags, 16) + __table_args__ = ( + Index('page_title', title), + Index('page_touched', touched), + UniqueConstraint(url, name='page_url'), + Index('page_calendar', calendar) + ) + revisions: Relationship[List[PageRevision]] = relationship("PageRevision", back_populates = 'page') owner: Relationship[User] = relationship('User', back_populates = 'owned_pages') tags: Relationship[List[PageTag]] = relationship('PageTag', back_populates = 'page') @@ -313,19 +326,15 @@ class Page(db.Model): class PageText(db.Model): __tablename__ = 'pagetext' - id = Column(Integer, primary_key=True) - content = Column(Blob) - flags = Column(BigInteger, default=0) - is_utf8 = BitSelector(flags, 1) - is_gzipped = BitSelector(flags, 2) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + content: Mapped[bytes] = mapped_column(Blob) + flags: Mapped[int] = mapped_column(BigInteger, default=0) + is_gzipped: Mapped[bool] = BitSelector(flags, 2) def get_content(self) -> str: c = self.content if self.is_gzipped: c = gzip.decompress(c) - if self.is_utf8: - return c.decode('utf-8') - else: - return c.decode('latin-1') + return c.decode('utf-8', 'replace') @classmethod def create_content(cls, text: str, *, treshold=600, search_dup = True) -> PageText: c: bytes = text.encode('utf-8') @@ -338,7 +347,6 @@ class PageText(db.Model): return item return db.session.execute(insert(cls).values( content = c, - is_utf8 = True, is_gzipped = use_gzip ).returning(cls)).scalar() @@ -347,16 +355,21 @@ class PageText(db.Model): class PageRevision(db.Model): __tablename__ = 'pagerevision' - id = Column(Integer, primary_key=True) - page_id = Column(Integer, ForeignKey('page.id')) - user_id = Column(Integer, ForeignKey('user.id'), nullable=True) - comment = Column(String(1024), default='') - textref_id = Column(Integer, ForeignKey('pagetext.id')) - pub_date = Column(DateTime, index=True) - length = Column(Integer) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + page_id: Mapped[int] = bound_fk(Page.id) + user_id: Mapped[int | None] = unbound_fk(User.id) + comment: Mapped[str] = mapped_column(String(1024), default='') + textref_id: Mapped[int] = bound_fk(PageText.id) + pub_date: Mapped[datetime.datetime] = mapped_column(DateTime, index=True) + length: Mapped[int] = mapped_column() - page = relationship("Page", back_populates='revisions') - user = relationship("User", back_populates='contributions') + __table_args__ = ( + Index('ix_pagerevision_pub_date', pub_date), + + ) + + page: Relationship[Page] = relationship("Page", back_populates='revisions') + user: Relationship[User] = relationship("User", back_populates='contributions') textref: Relationship[PageText] = relationship("PageText") @@ -385,12 +398,13 @@ class PageRevision(db.Model): class PageTag(db.Model): __tablename__ = 'pagetag' __table_args__ = ( - UniqueConstraint('page_id', 'name'), + UniqueConstraint('page_id', 'name', name='pagetag_page_id_name'), + Index('ix_pagetag_name', 'name') ) - id = Column(Integer, primary_key=True) - page_id = Column(Integer, ForeignKey('page.id')) - name: Column[str] = Column(String(64), index=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + page_id: Mapped[int] = bound_fk(Page.id) + name: Mapped[str] = mapped_column(String(64), index=True) page = relationship('Page', back_populates='tags') @@ -401,20 +415,24 @@ class PageTag(db.Model): class PageProperty(db.Model): + """ + XXX as of 1.1.0, PageProperty is unused + """ __tablename__ = 'pageproperty' __table_args__ = ( - UniqueConstraint('page_id', 'key'), + UniqueConstraint('page_id', 'key', name="pageproperty_page_id_key"), ) - id = Column(Integer, primary_key = True) - page_id = Column(Integer, ForeignKey('page.id')) - key = Column(String(64)) - value = Column(String(8000)) + id: Mapped[int] = mapped_column(Integer, primary_key = True) + page_id: Mapped[int] = mapped_column(Integer, ForeignKey('page.id')) + key: Mapped[str] = mapped_column(String(64)) + value: Mapped[str] = mapped_column(String(8000)) page = relationship('Page', back_populates = 'properties') -# XXX is it *really* worth it to implement PagePropertyDict? +# XXX PagePropertyDict? +@deprecated('is it *really* worth it to implement?') class _PagePropertyDict(Mapping): __slots__ = ('_page', ) def __init__(self, page: Page, /) -> tuple[str, Any]: @@ -456,18 +474,21 @@ class _PagePropertyDict(Mapping): else: return self._page.get_prop(key) + class PageLink(db.Model): __tablename__ = 'pagelink' - __table_args__ = ( - UniqueConstraint('from_page_id', 'to_page_id'), - ) - id = Column(Integer, primary_key = True) - from_page_id = Column(Integer, ForeignKey('page.id')) - to_page_id = Column(Integer, ForeignKey('page.id')) + id: Mapped[int] = mapped_column(Integer, primary_key = True) + from_page_id: Mapped[int] = bound_fk(Page.id) + to_page_id: Mapped[int] = bound_fk(Page.id) + + __table_args__ = ( + UniqueConstraint('from_page_id', 'to_page_id', name='pagelink_from_page_id_to_page_id'), + Index('to_page_id', to_page_id) + ) from_page: Relationship[Page] = relationship('Page', foreign_keys=[from_page_id], back_populates='forward_links') - to_page: Relationship[Page] = relationship('Page', foreign_keys=[to_page_id], back_populates='back_links') + to_page: Relationship[Page] = relationship('Page', foreign_keys=[to_page_id], back_populates='back_links') @classmethod def parse_links(cls, from_page, text: str, erase=True): @@ -507,19 +528,22 @@ class PageLink(db.Model): class PagePermission(db.Model): __tablename__ = 'pagepermission' - __table_args__ = ( - UniqueConstraint('page_id', 'group_id'), - ) - id = Column(Integer, primary_key = True) - page_id = Column(Integer, ForeignKey('page.id')) - group_id = Column(Integer, ForeignKey('usergroup.id')) - permissions = Column(BigInteger, default=0) - can_read = BitSelector(permissions, PERM_READ) - can_edit = BitSelector(permissions, PERM_EDIT) - can_create = BitSelector(permissions, PERM_CREATE) - can_set_url = BitSelector(permissions, PERM_SET_URL) - can_set_tags = BitSelector(permissions, PERM_SET_TAGS) + id: Mapped[int] = mapped_column(Integer, primary_key = True) + page_id: Mapped[int] = bound_fk(Page.id) + group_id: Mapped[int] = bound_fk(UserGroup.id) + # v--- will be split in 1.2, otherwise sooner or later + permissions: Mapped[int] = mapped_column(BigInteger, server_default=text('0')) + can_read: Mapped[bool] = BitSelector(permissions, PERM_READ) + can_edit: Mapped[bool] = BitSelector(permissions, PERM_EDIT) + can_create: Mapped[bool] = BitSelector(permissions, PERM_CREATE) + can_set_url: Mapped[bool] = BitSelector(permissions, PERM_SET_URL) + can_set_tags: Mapped[bool] = BitSelector(permissions, PERM_SET_TAGS) + + __table_args__ = ( + UniqueConstraint('page_id', 'group_id', name="pagepermission_page_id_group_id"), + Index('group_id', group_id) + ) page = relationship('Page', back_populates = 'permission_overrides') group = relationship('UserGroup', back_populates = 'page_permissions') diff --git a/salvi/routes/edit.py b/salvi/routes/edit.py index 4be78de..0901229 100644 --- a/salvi/routes/edit.py +++ b/salvi/routes/edit.py @@ -14,7 +14,7 @@ from salvi.i18n import get_string from ..utils import parse_tag_list from ..renderer import md_and_toc -from ..models import PERM_CREATE, Page, PageLink, PageRevision, PageText, User, db, is_url_available, is_valid_url, perms_required +from ..models import PERM_CREATE, PERM_EDIT, Page, PageLink, PageRevision, PageText, User, db, is_url_available, is_valid_url, perms_required current_user: User @@ -110,7 +110,7 @@ def create(): return savepoint(dict(url=request.args.get('url'), title='', text='', tags='', comment=get_string(g.lang, 'page_created'))) @bp.route('/edit/', methods=['GET', 'POST']) -@login_required +@perms_required(PERM_EDIT, message='You are not authorized to edit pages.') def edit(id: int): p = db.session.execute(select(Page).where(Page.id == id)).scalar() if p is None: