diff options
Diffstat (limited to 'server')
31 files changed, 301 insertions, 166 deletions
diff --git a/server/Dockerfile b/server/Dockerfile index 4beec1c..3e4dadf 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -7,8 +7,13 @@ WORKDIR /opt/app RUN apk --no-cache add \ python3 \ python3-dev \ - ffmpeg \ py3-pip \ + build-base \ + libheif \ + libheif-dev \ + libavif \ + libavif-dev \ + ffmpeg \ # from requirements.txt: py3-yaml \ py3-psycopg2 \ @@ -18,26 +23,21 @@ RUN apk --no-cache add \ py3-pillow \ py3-pynacl \ py3-tz \ - py3-pyrfc3339 \ - build-base \ - && apk --no-cache add \ - libheif \ - libavif \ - libheif-dev \ - libavif-dev \ - && pip3 install --no-cache-dir --disable-pip-version-check \ - alembic \ + py3-pyrfc3339 +RUN pip3 install --no-cache-dir --disable-pip-version-check \ + "alembic>=0.8.5" \ "coloredlogs==5.0" \ - youtube_dl \ - pillow-avif-plugin \ - pyheif-pillow-opener \ - && apk --no-cache del py3-pip + "pyheif==0.6.1" \ + "heif-image-plugin>=0.3.2" \ + yt-dlp \ + "pillow-avif-plugin~=1.1.0" +RUN apk --no-cache del py3-pip COPY ./ /opt/app/ RUN rm -rf /opt/app/szurubooru/tests -FROM prereqs as testing +FROM --platform=$BUILDPLATFORM prereqs as testing WORKDIR /opt/app RUN apk --no-cache add \ @@ -83,6 +83,9 @@ ARG PORT=6666 ENV PORT=${PORT} EXPOSE ${PORT} +ARG THREADS=4 +ENV THREADS=${THREADS} + VOLUME ["/data/"] ARG DOCKER_REPO diff --git a/server/config.yaml.dist b/server/config.yaml.dist index 3a17e38..5dddd72 100644 --- a/server/config.yaml.dist +++ b/server/config.yaml.dist @@ -116,6 +116,7 @@ privileges: 'posts:bulk-edit:tags': power 'posts:bulk-edit:safety': power 'posts:view:similar': regular + 'posts:bulk-edit:delete': power 'tags:create': regular 'tags:edit:names': power diff --git a/server/docker-start.sh b/server/docker-start.sh index 34a0e49..eebef1c 100755 --- a/server/docker-start.sh +++ b/server/docker-start.sh @@ -4,5 +4,5 @@ cd /opt/app alembic upgrade head -echo "Starting szurubooru API on port ${PORT}" -exec waitress-serve-3 --port ${PORT} szurubooru.facade:app +echo "Starting szurubooru API on port ${PORT} - Running on ${THREADS} threads" +exec waitress-serve-3 --port ${PORT} --threads ${THREADS} szurubooru.facade:app diff --git a/server/hooks/build b/server/hooks/build deleted file mode 100755 index b5e914b..0000000 --- a/server/hooks/build +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -docker build \ - --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - --build-arg SOURCE_COMMIT \ - --build-arg DOCKER_REPO \ - -f $DOCKERFILE_PATH -t $IMAGE_NAME . diff --git a/server/hooks/post_push b/server/hooks/post_push deleted file mode 100755 index 1b1e0ad..0000000 --- a/server/hooks/post_push +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -add_tag() { - echo "Also tagging image as ${DOCKER_REPO}:${1}" - docker tag $IMAGE_NAME $DOCKER_REPO:$1 - docker push $DOCKER_REPO:$1 -} - -CLOSEST_VER=$(git describe --tags --abbrev=0) -CLOSEST_MAJOR_VER=$(echo ${CLOSEST_VER} | cut -d'.' -f1) -CLOSEST_MINOR_VER=$(echo ${CLOSEST_VER} | cut -d'.' -f2) - -add_tag "${CLOSEST_MAJOR_VER}-edge" -add_tag "${CLOSEST_MAJOR_VER}.${CLOSEST_MINOR_VER}-edge" - -if git describe --exact-match --abbrev=0 2> /dev/null; then - add_tag "${CLOSEST_MAJOR_VER}" - add_tag "${CLOSEST_MAJOR_VER}.${CLOSEST_MINOR_VER}" -fi diff --git a/server/hooks/test b/server/hooks/test deleted file mode 100755 index b325186..0000000 --- a/server/hooks/test +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e - -docker run --rm \ - -t $(docker build --target testing -q .) \ - --color=no szurubooru/ - -exit $? diff --git a/server/requirements.txt b/server/requirements.txt index 2a09b24..ffe18f0 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,14 +1,15 @@ alembic>=0.8.5 -pyyaml>=3.11 -psycopg2-binary>=2.6.1 -SQLAlchemy>=1.0.12, <1.4 -coloredlogs==5.0 certifi>=2017.11.5 +coloredlogs==5.0 +heif-image-plugin==0.3.2 numpy>=1.8.2 +pillow-avif-plugin~=1.1.0 pillow>=4.3.0 +psycopg2-binary>=2.6.1 +pyheif==0.6.1 pynacl>=1.2.1 -pytz>=2018.3 pyRFC3339>=1.0 -pillow-avif-plugin>=1.1.0 -pyheif-pillow-opener>=0.1.0 -youtube_dl +pytz>=2018.3 +pyyaml>=3.11 +SQLAlchemy>=1.0.12, <1.4 +yt-dlp diff --git a/server/szuru-admin b/server/szuru-admin index 004a751..08ba182 100755 --- a/server/szuru-admin +++ b/server/szuru-admin @@ -91,6 +91,15 @@ def reset_filenames() -> None: rename_in_dir("posts/custom-thumbnails/") +def regenerate_thumbnails() -> None: + for post in db.session.query(model.Post).all(): + print("Generating tumbnail for post %d ..." % post.post_id, end="\r") + try: + postfuncs.generate_post_thumbnail(post) + except Exception: + pass + + def main() -> None: parser_top = ArgumentParser( description="Collection of CLI commands for an administrator to use", @@ -114,6 +123,12 @@ def main() -> None: help="reset and rename the content and thumbnail " "filenames in case of a lost/changed secret key", ) + parser.add_argument( + "--regenerate-thumbnails", + action="store_true", + help="regenerate the thumbnails for posts if the " + "thumbnail files are missing", + ) command = parser_top.parse_args() try: @@ -123,6 +138,8 @@ def main() -> None: check_audio() elif command.reset_filenames: reset_filenames() + elif command.regenerate_thumbnails: + regenerate_thumbnails() except errors.BaseError as e: print(e, file=stderr) diff --git a/server/szurubooru/config.py b/server/szurubooru/config.py index 1515a54..f3f9007 100644 --- a/server/szurubooru/config.py +++ b/server/szurubooru/config.py @@ -21,7 +21,7 @@ def _merge(left: Dict, right: Dict) -> Dict: return left -def _docker_config() -> Dict: +def _container_config() -> Dict: if "TEST_ENVIRONMENT" not in os.environ: for key in ["POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_HOST"]: if key not in os.environ: @@ -33,7 +33,7 @@ def _docker_config() -> Dict: "show_sql": int(os.getenv("LOG_SQL", 0)), "data_url": os.getenv("DATA_URL", "data/"), "data_dir": "/data/", - "database": "postgres://%(user)s:%(pass)s@%(host)s:%(port)d/%(db)s" + "database": "postgresql://%(user)s:%(pass)s@%(host)s:%(port)d/%(db)s" % { "user": os.getenv("POSTGRES_USER"), "pass": os.getenv("POSTGRES_PASSWORD"), @@ -49,6 +49,15 @@ def _file_config(filename: str) -> Dict: return yaml.load(handle.read(), Loader=yaml.SafeLoader) or {} +def _running_inside_container() -> bool: + env = os.environ.keys() + return ( + os.path.exists("/.dockerenv") + or "KUBERNETES_SERVICE_HOST" in env + or "container" in env # set by lxc/podman + ) + + def _read_config() -> Dict: ret = _file_config("config.yaml.dist") if os.path.isfile("config.yaml"): @@ -57,8 +66,8 @@ def _read_config() -> Dict: logger.warning( "'config.yaml' should be a file, not a directory, skipping" ) - if os.path.exists("/.dockerenv"): - ret = _merge(ret, _docker_config()) + if _running_inside_container(): + ret = _merge(ret, _container_config()) return ret diff --git a/server/szurubooru/facade.py b/server/szurubooru/facade.py index a7e4844..4c8084f 100644 --- a/server/szurubooru/facade.py +++ b/server/szurubooru/facade.py @@ -135,7 +135,7 @@ _live_migrations = ( def create_app() -> Callable[[Any, Any], Any]: - """ Create a WSGI compatible App object. """ + """Create a WSGI compatible App object.""" validate_config() coloredlogs.install(fmt="[%(asctime)-15s] %(name)s %(message)s") if config.config["debug"]: diff --git a/server/szurubooru/func/auth.py b/server/szurubooru/func/auth.py index d013775..17d25f7 100644 --- a/server/szurubooru/func/auth.py +++ b/server/szurubooru/func/auth.py @@ -25,7 +25,7 @@ RANK_MAP = OrderedDict( def get_password_hash(salt: str, password: str) -> Tuple[str, int]: - """ Retrieve argon2id password hash. """ + """Retrieve argon2id password hash.""" return ( pwhash.argon2id.str( (config.config["secret"] + salt + password).encode("utf8") @@ -37,7 +37,7 @@ def get_password_hash(salt: str, password: str) -> Tuple[str, int]: def get_sha256_legacy_password_hash( salt: str, password: str ) -> Tuple[str, int]: - """ Retrieve old-style sha256 password hash. """ + """Retrieve old-style sha256 password hash.""" digest = hashlib.sha256() digest.update(config.config["secret"].encode("utf8")) digest.update(salt.encode("utf8")) @@ -46,7 +46,7 @@ def get_sha256_legacy_password_hash( def get_sha1_legacy_password_hash(salt: str, password: str) -> Tuple[str, int]: - """ Retrieve old-style sha1 password hash. """ + """Retrieve old-style sha1 password hash.""" digest = hashlib.sha1() digest.update(b"1A2/$_4xVa") digest.update(salt.encode("utf8")) @@ -125,7 +125,7 @@ def verify_privilege(user: model.User, privilege_name: str) -> None: def generate_authentication_token(user: model.User) -> str: - """ Generate nonguessable challenge (e.g. links in password reminder). """ + """Generate nonguessable challenge (e.g. links in password reminder).""" assert user digest = hashlib.md5() digest.update(config.config["secret"].encode("utf8")) diff --git a/server/szurubooru/func/image_hash.py b/server/szurubooru/func/image_hash.py index a445e62..76d5a84 100644 --- a/server/szurubooru/func/image_hash.py +++ b/server/szurubooru/func/image_hash.py @@ -4,12 +4,10 @@ from datetime import datetime from io import BytesIO from typing import Any, Callable, List, Optional, Set, Tuple +import HeifImagePlugin import numpy as np -from PIL import Image import pillow_avif -import pyheif -from pyheif_pillow_opener import register_heif_opener -register_heif_opener() +from PIL import Image from szurubooru import config, errors diff --git a/server/szurubooru/func/images.py b/server/szurubooru/func/images.py index 101bba8..e135d18 100644 --- a/server/szurubooru/func/images.py +++ b/server/szurubooru/func/images.py @@ -6,6 +6,9 @@ import shlex import subprocess from io import BytesIO from typing import List + +import HeifImagePlugin +import pillow_avif from PIL import Image as PILImage from szurubooru import errors @@ -17,7 +20,7 @@ logger = logging.getLogger(__name__) def convert_heif_to_png(content: bytes) -> bytes: img = PILImage.open(BytesIO(content)) img_byte_arr = BytesIO() - img.save(img_byte_arr, format='PNG') + img.save(img_byte_arr, format="PNG") return img_byte_arr.getvalue() @@ -276,10 +279,10 @@ class Image: proc = subprocess.Popen( cli, stdout=subprocess.PIPE, - stdin=subprocess.PIPE, + stdin=subprocess.DEVNULL, stderr=subprocess.PIPE, ) - out, err = proc.communicate(input=self.content) + out, err = proc.communicate() if proc.returncode != 0: logger.warning( "Failed to execute ffmpeg command (cli=%r, err=%r)", diff --git a/server/szurubooru/func/mime.py b/server/szurubooru/func/mime.py index 93c096b..8fae567 100644 --- a/server/szurubooru/func/mime.py +++ b/server/szurubooru/func/mime.py @@ -36,9 +36,12 @@ def get_mime_type(content: bytes) -> str: if content[0:4] == b"\x1A\x45\xDF\xA3": return "video/webm" - if content[4:12] in (b"ftypisom", b"ftypiso5", b"ftypmp42", b"ftypM4V "): + if content[4:12] in (b"ftypisom", b"ftypiso5", b"ftypiso6", b"ftypmp42", b"ftypM4V "): return "video/mp4" + if content[4:12] == b"ftypqt ": + return "video/quicktime" + return "application/octet-stream" @@ -54,6 +57,7 @@ def get_extension(mime_type: str) -> Optional[str]: "image/heif": "heif", "image/heic": "heic", "video/mp4": "mp4", + "video/quicktime": "mov", "video/webm": "webm", "application/octet-stream": "dat", } @@ -65,7 +69,12 @@ def is_flash(mime_type: str) -> bool: def is_video(mime_type: str) -> bool: - return mime_type.lower() in ("application/ogg", "video/mp4", "video/webm") + return mime_type.lower() in ( + "application/ogg", + "video/mp4", + "video/quicktime", + "video/webm", + ) def is_image(mime_type: str) -> bool: @@ -88,6 +97,7 @@ def is_animated_gif(content: bytes) -> bool: and len(re.findall(pattern, content)) > 1 ) + def is_heif(mime_type: str) -> bool: return mime_type.lower() in ( "image/heif", diff --git a/server/szurubooru/func/net.py b/server/szurubooru/func/net.py index 9dff3c4..d6aa95e 100644 --- a/server/szurubooru/func/net.py +++ b/server/szurubooru/func/net.py @@ -39,13 +39,20 @@ def download(url: str, use_video_downloader: bool = False) -> bytes: length_tally = 0 try: with urllib.request.urlopen(request) as handle: - while (chunk := handle.read(_dl_chunk_size)) : + while chunk := handle.read(_dl_chunk_size): length_tally += len(chunk) if length_tally > config.config["max_dl_filesize"]: - raise DownloadTooLargeError(url) + raise DownloadTooLargeError( + "Download target exceeds maximum. (%d)" + % (config.config["max_dl_filesize"]), + extra_fields={"URL": url}, + ) content_buffer += chunk except urllib.error.HTTPError as ex: - raise DownloadError(url) from ex + raise DownloadError( + "Download target returned HTTP %d. (%s)" % (ex.code, ex.reason), + extra_fields={"URL": url}, + ) from ex if ( youtube_dl_error @@ -57,7 +64,7 @@ def download(url: str, use_video_downloader: bool = False) -> bytes: def _get_youtube_dl_content_url(url: str) -> str: - cmd = ["youtube-dl", "--format", "best", "--no-playlist"] + cmd = ["yt-dlp", "--format", "best", "--no-playlist"] if config.config["user_agent"]: cmd.extend(["--user-agent", config.config["user_agent"]]) cmd.extend(["--get-url", url]) @@ -69,7 +76,8 @@ def _get_youtube_dl_content_url(url: str) -> str: ) except subprocess.CalledProcessError: raise errors.ThirdPartyError( - "Could not extract content location from %s" % (url) + "Could not extract content location from URL.", + extra_fields={"URL": url}, ) from None diff --git a/server/szurubooru/func/util.py b/server/szurubooru/func/util.py index eacdc2a..dc5ced0 100644 --- a/server/szurubooru/func/util.py +++ b/server/szurubooru/func/util.py @@ -87,12 +87,12 @@ def flip(source: Dict[Any, Any]) -> Dict[Any, Any]: def is_valid_email(email: Optional[str]) -> bool: - """ Return whether given email address is valid or empty. """ + """Return whether given email address is valid or empty.""" return not email or re.match(r"^[^@]*@[^@]*\.[^@]*$", email) is not None class dotdict(dict): - """ dot.notation access to dictionary attributes. """ + """dot.notation access to dictionary attributes.""" def __getattr__(self, attr: str) -> Any: return self.get(attr) @@ -102,7 +102,7 @@ class dotdict(dict): def parse_time_range(value: str) -> Tuple[datetime, datetime]: - """ Return tuple containing min/max time for given text representation. """ + """Return tuple containing min/max time for given text representation.""" one_day = timedelta(days=1) one_second = timedelta(seconds=1) almost_one_day = one_day - one_second diff --git a/server/szurubooru/middleware/authenticator.py b/server/szurubooru/middleware/authenticator.py index e73b235..436543b 100644 --- a/server/szurubooru/middleware/authenticator.py +++ b/server/szurubooru/middleware/authenticator.py @@ -7,7 +7,7 @@ from szurubooru.rest.errors import HttpBadRequest def _authenticate_basic_auth(username: str, password: str) -> model.User: - """ Try to authenticate user. Throw AuthError for invalid users. """ + """Try to authenticate user. Throw AuthError for invalid users.""" user = users.get_user_by_name(username) if not auth.is_valid_password(user, password): raise errors.AuthError("Invalid password.") @@ -17,7 +17,7 @@ def _authenticate_basic_auth(username: str, password: str) -> model.User: def _authenticate_token( username: str, token: str ) -> Tuple[model.User, model.UserToken]: - """ Try to authenticate user. Throw AuthError for invalid users. """ + """Try to authenticate user. Throw AuthError for invalid users.""" user = users.get_user_by_name(username) user_token = user_tokens.get_by_user_and_token(user, token) if not auth.is_valid_token(user_token): @@ -72,7 +72,7 @@ def _get_user(ctx: rest.Context, bump_login: bool) -> Optional[model.User]: def process_request(ctx: rest.Context) -> None: - """ Bind the user to request. Update last login time if needed. """ + """Bind the user to request. Update last login time if needed.""" bump_login = ctx.get_param_as_bool("bump-login", default=False) auth_user = _get_user(ctx, bump_login) if auth_user: diff --git a/server/szurubooru/rest/app.py b/server/szurubooru/rest/app.py index a6f10fb..c098bd0 100644 --- a/server/szurubooru/rest/app.py +++ b/server/szurubooru/rest/app.py @@ -11,7 +11,7 @@ from szurubooru.rest import context, errors, middleware, routes def _json_serializer(obj: Any) -> str: - """ JSON serializer for objects not serializable by default JSON code """ + """JSON serializer for objects not serializable by default JSON code""" if isinstance(obj, datetime): serial = obj.isoformat("T") + "Z" return serial diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py index 4b2c8bb..9869843 100644 --- a/server/szurubooru/search/configs/post_search_config.py +++ b/server/szurubooru/search/configs/post_search_config.py @@ -214,6 +214,34 @@ def _create_metric_sort_column(metric_name: str): return ret +def _category_filter( + query: SaQuery, criterion: Optional[criteria.BaseCriterion], negated: bool +) -> SaQuery: + assert criterion + + # Step 1. find the id for the category + q1 = db.session.query(model.TagCategory.tag_category_id).filter( + model.TagCategory.name == criterion.value + ) + + # Step 2. find the tags with that category + q2 = db.session.query(model.Tag.tag_id).filter( + model.Tag.category_id.in_(q1) + ) + + # Step 3. find all posts that have at least one of those tags + q3 = db.session.query(model.PostTag.post_id).filter( + model.PostTag.tag_id.in_(q2) + ) + + # Step 4. profit + expr = model.Post.post_id.in_(q3) + if negated: + expr = ~expr + + return query.filter(expr) + + class PostSearchConfig(BaseSearchConfig): def __init__(self) -> None: self.user = None # type: Optional[model.User] @@ -451,6 +479,7 @@ class PostSearchConfig(BaseSearchConfig): ), (["pool"], _pool_filter), (["similar"], _similar_filter), + (["category"], _category_filter), ] )) return filters diff --git a/server/szurubooru/tests/api/test_tag_updating.py b/server/szurubooru/tests/api/test_tag_updating.py index 9112c29..66939a4 100644 --- a/server/szurubooru/tests/api/test_tag_updating.py +++ b/server/szurubooru/tests/api/test_tag_updating.py @@ -167,8 +167,9 @@ def test_trying_to_create_metric_without_privileges( ) +@pytest.mark.parametrize("type", ["suggestions", "implications"]) def test_trying_to_create_tags_without_privileges( - config_injector, context_factory, tag_factory, user_factory + config_injector, context_factory, tag_factory, user_factory, type ): tag = tag_factory(names=["tag"]) db.session.add(tag) @@ -187,16 +188,7 @@ def test_trying_to_create_tags_without_privileges( with pytest.raises(errors.AuthError): api.tag_api.update_tag( context_factory( - params={"suggestions": ["tag1", "tag2"], "version": 1}, - user=user_factory(rank=model.User.RANK_REGULAR), - ), - {"tag_name": "tag"}, - ) - db.session.rollback() - with pytest.raises(errors.AuthError): - api.tag_api.update_tag( - context_factory( - params={"implications": ["tag1", "tag2"], "version": 1}, + params={type: ["tag1", "tag2"], "version": 1}, user=user_factory(rank=model.User.RANK_REGULAR), ), {"tag_name": "tag"}, diff --git a/server/szurubooru/tests/assets/mov.mov b/server/szurubooru/tests/assets/mov.mov Binary files differnew file mode 100644 index 0000000..911ee85 --- /dev/null +++ b/server/szurubooru/tests/assets/mov.mov diff --git a/server/szurubooru/tests/conftest.py b/server/szurubooru/tests/conftest.py index 50cbf7d..45113f5 100644 --- a/server/szurubooru/tests/conftest.py +++ b/server/szurubooru/tests/conftest.py @@ -43,14 +43,26 @@ def query_logger(pytestconfig): logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) -@pytest.yield_fixture(scope="function", autouse=True) -def session(query_logger, postgresql_db): +@pytest.fixture(scope="function", autouse=True) +def session(query_logger, transacted_postgresql_db): + db.session = transacted_postgresql_db.session + transacted_postgresql_db.create_table(*model.Base.metadata.sorted_tables) + try: + yield transacted_postgresql_db.session + finally: + transacted_postgresql_db.reset_db() + + +@pytest.fixture(scope="function") +def nontransacted_session(query_logger, postgresql_db): + old_db_session = db.session db.session = postgresql_db.session postgresql_db.create_table(*model.Base.metadata.sorted_tables) try: yield postgresql_db.session finally: postgresql_db.reset_db() + db.session = old_db_session @pytest.fixture diff --git a/server/szurubooru/tests/func/test_mime.py b/server/szurubooru/tests/func/test_mime.py index b33746b..551ba7c 100644 --- a/server/szurubooru/tests/func/test_mime.py +++ b/server/szurubooru/tests/func/test_mime.py @@ -7,6 +7,7 @@ from szurubooru.func import mime "input_path,expected_mime_type", [ ("mp4.mp4", "video/mp4"), + ("mov.mov", "video/quicktime"), ("webm.webm", "video/webm"), ("flash.swf", "application/x-shockwave-flash"), ("png.png", "image/png"), @@ -35,6 +36,7 @@ def test_get_mime_type_for_empty_file(): [ ("video/mp4", "mp4"), ("video/webm", "webm"), + ("video/quicktime", "mov"), ("application/x-shockwave-flash", "swf"), ("image/png", "png"), ("image/jpeg", "jpg"), @@ -70,6 +72,8 @@ def test_is_flash(input_mime_type, expected_state): ("VIDEO/WEBM", True), ("video/mp4", True), ("VIDEO/MP4", True), + ("video/quicktime", True), + ("VIDEO/QUICKTIME", True), ("video/anything_else", False), ("application/ogg", True), ("not a video", False), diff --git a/server/szurubooru/tests/func/test_net.py b/server/szurubooru/tests/func/test_net.py index c5b4c73..be2f3c9 100644 --- a/server/szurubooru/tests/func/test_net.py +++ b/server/szurubooru/tests/func/test_net.py @@ -1,3 +1,5 @@ +import os + import pytest from szurubooru import errors @@ -16,6 +18,9 @@ def inject_config(tmpdir, config_injector): ) +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) def test_download(): url = "http://info.cern.ch/hypertext/WWW/TheProject.html" @@ -62,6 +67,9 @@ def test_download(): assert actual_content == expected_content +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) @pytest.mark.parametrize( "url", [ @@ -74,6 +82,9 @@ def test_too_large_download(url): net.download(url, use_video_downloader=True) +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) @pytest.mark.parametrize( "url,expected_sha1", [ @@ -96,6 +107,9 @@ def test_content_download(url, expected_sha1): assert get_sha1(actual_content) == expected_sha1 +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) def test_bad_content_downlaod(): url = "http://info.cern.ch/hypertext/WWW/TheProject.html" with pytest.raises(errors.ThirdPartyError): @@ -108,11 +122,13 @@ def test_no_webhooks(config_injector): assert len(res) == 0 +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) @pytest.mark.parametrize( "webhook,status_code", [ ("https://postman-echo.com/post", 200), - ("http://localhost/", 400), ("https://postman-echo.com/get", 400), ], ) @@ -121,6 +137,9 @@ def test_single_webhook(config_injector, webhook, status_code): assert ret == status_code +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) def test_multiple_webhooks(config_injector): config_injector( { diff --git a/server/szurubooru/tests/func/test_snapshots.py b/server/szurubooru/tests/func/test_snapshots.py index da93530..dc68ff0 100644 --- a/server/szurubooru/tests/func/test_snapshots.py +++ b/server/szurubooru/tests/func/test_snapshots.py @@ -1,7 +1,7 @@ from datetime import datetime from unittest.mock import patch -import pytest +import pytest # noqa: F401 from szurubooru import db, model from szurubooru.func import snapshots, users @@ -144,46 +144,6 @@ def test_create(tag_factory, user_factory): assert results[0].data == "mocked" -def test_modify_saves_non_empty_diffs(post_factory, user_factory): - if "sqlite" in db.session.get_bind().driver: - pytest.xfail( - "SQLite doesn't support transaction isolation, " - "which is required to retrieve original entity" - ) - post = post_factory() - post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="old")] - user = user_factory() - db.session.add_all([post, user]) - db.session.commit() - post.source = "new source" - post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="new")] - db.session.flush() - with patch("szurubooru.func.snapshots._post_to_webhooks"): - snapshots.modify(post, user) - db.session.flush() - results = db.session.query(model.Snapshot).all() - assert len(results) == 1 - assert results[0].data == { - "type": "object change", - "value": { - "source": { - "type": "primitive change", - "old-value": None, - "new-value": "new source", - }, - "notes": { - "type": "list change", - "removed": [ - {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"} - ], - "added": [ - {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"} - ], - }, - }, - } - - def test_modify_doesnt_save_empty_diffs(tag_factory, user_factory): tag = tag_factory(names=["dummy"]) user = user_factory() diff --git a/server/szurubooru/tests/func/test_snapshots_transactional_isolation.py b/server/szurubooru/tests/func/test_snapshots_transactional_isolation.py new file mode 100644 index 0000000..b98cea7 --- /dev/null +++ b/server/szurubooru/tests/func/test_snapshots_transactional_isolation.py @@ -0,0 +1,59 @@ +from unittest.mock import patch + +import pytest + +from szurubooru import db, model +from szurubooru.func import snapshots + + +@pytest.fixture(autouse=True) +def session(query_logger, postgresql_db): + """ + Override db session for this specific test section only + """ + db.session = postgresql_db.session + postgresql_db.create_table(*model.Base.metadata.sorted_tables) + try: + yield postgresql_db.session + finally: + postgresql_db.reset_db() + + +def test_modify_saves_non_empty_diffs(post_factory, user_factory): + if "sqlite" in db.session.get_bind().driver: + pytest.xfail( + "SQLite doesn't support transaction isolation, " + "which is required to retrieve original entity" + ) + post = post_factory() + post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="old")] + user = user_factory() + db.session.add_all([post, user]) + db.session.commit() + post.source = "new source" + post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="new")] + db.session.flush() + with patch("szurubooru.func.snapshots._post_to_webhooks"): + snapshots.modify(post, user) + db.session.flush() + results = db.session.query(model.Snapshot).all() + assert len(results) == 1 + assert results[0].data == { + "type": "object change", + "value": { + "source": { + "type": "primitive change", + "old-value": None, + "new-value": "new source", + }, + "notes": { + "type": "list change", + "removed": [ + {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"} + ], + "added": [ + {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"} + ], + }, + }, + } diff --git a/server/szurubooru/tests/func/test_tag_categories.py b/server/szurubooru/tests/func/test_tag_categories.py index 11300cf..9e649a3 100644 --- a/server/szurubooru/tests/func/test_tag_categories.py +++ b/server/szurubooru/tests/func/test_tag_categories.py @@ -107,17 +107,16 @@ def test_update_category_name_reusing_other_name( tag_categories.update_category_name(category, "NAME") +@pytest.mark.parametrize("name", ["name", "NAME"]) def test_update_category_name_reusing_own_name( - config_injector, tag_category_factory + config_injector, tag_category_factory, name ): config_injector({"tag_category_name_regex": ".*"}) - for name in ["name", "NAME"]: - category = tag_category_factory(name="name") - db.session.add(category) - db.session.flush() - tag_categories.update_category_name(category, name) - assert category.name == name - db.session.rollback() + category = tag_category_factory(name="name") + db.session.add(category) + db.session.flush() + tag_categories.update_category_name(category, name) + assert category.name == name def test_update_category_color_with_empty_string(tag_category_factory): diff --git a/server/szurubooru/tests/func/test_tags.py b/server/szurubooru/tests/func/test_tags.py index c938e68..79376f4 100644 --- a/server/szurubooru/tests/func/test_tags.py +++ b/server/szurubooru/tests/func/test_tags.py @@ -541,15 +541,14 @@ def test_update_tag_names_trying_to_use_taken_name( tags.update_tag_names(tag, ["A"]) -def test_update_tag_names_reusing_own_name(config_injector, tag_factory): +@pytest.mark.parametrize("name", list("aA")) +def test_update_tag_names_reusing_own_name(config_injector, tag_factory, name): config_injector({"tag_name_regex": "^[a-zA-Z]*$"}) - for name in list("aA"): - tag = tag_factory(names=["a"]) - db.session.add(tag) - db.session.flush() - tags.update_tag_names(tag, [name]) - assert [tag_name.name for tag_name in tag.names] == [name] - db.session.rollback() + tag = tag_factory(names=["a"]) + db.session.add(tag) + db.session.flush() + tags.update_tag_names(tag, [name]) + assert [tag_name.name for tag_name in tag.names] == [name] def test_update_tag_names_changing_primary_name(config_injector, tag_factory): @@ -561,7 +560,6 @@ def test_update_tag_names_changing_primary_name(config_injector, tag_factory): db.session.flush() db.session.refresh(tag) assert [tag_name.name for tag_name in tag.names] == ["b", "a"] - db.session.rollback() @pytest.mark.parametrize("attempt", ["name", "NAME", "alias", "ALIAS"]) diff --git a/server/szurubooru/tests/search/configs/test_pool_search_config.py b/server/szurubooru/tests/search/configs/test_pool_search_config.py index 202635c..1103ec4 100644 --- a/server/szurubooru/tests/search/configs/test_pool_search_config.py +++ b/server/szurubooru/tests/search/configs/test_pool_search_config.py @@ -136,8 +136,6 @@ def test_escaping( ) db.session.flush() - if db_driver and db.session.get_bind().driver != db_driver: - pytest.xfail() if expected_pool_names is None: with pytest.raises(errors.SearchError): executor.execute(input, offset=0, limit=100) diff --git a/server/szurubooru/tests/search/configs/test_post_search_config.py b/server/szurubooru/tests/search/configs/test_post_search_config.py index c9f2408..6ab5d52 100644 --- a/server/szurubooru/tests/search/configs/test_post_search_config.py +++ b/server/szurubooru/tests/search/configs/test_post_search_config.py @@ -1003,7 +1003,6 @@ def test_filter_by_similar( verify_unpaged(input, expected_post_ids, True) - @pytest.mark.parametrize("input,expected_post_ids", [ ("similar:1", [3, 1, 2]), ("similar:2", [3, 2, 1]), @@ -1023,3 +1022,55 @@ def test_sort_by_similar( ) db.session.flush() verify_unpaged(input, expected_post_ids, True) + + +@pytest.mark.parametrize( + "input,expected_post_ids", + [ + ("category:cat1", [1, 2, 3]), + ("category:cat2", [3, 4]), + ], +) +def test_search_by_tag_category( + verify_unpaged, + post_factory, + tag_factory, + tag_category_factory, + input, + expected_post_ids, +): + cat1 = tag_category_factory(name="cat1") + cat2 = tag_category_factory(name="cat2") + tag1 = tag_factory(names=["t1"], category=cat1) + tag2 = tag_factory(names=["t2"], category=cat1) + tag3 = tag_factory(names=["t3"], category=cat2) + + post1 = post_factory(id=1) + post1.tags.append(tag1) + + post2 = post_factory(id=2) + post2.tags.append(tag2) + + post3 = post_factory(id=3) + post3.tags.append(tag1) + post3.tags.append(tag3) + + post4 = post_factory(id=4) + post4.tags.append(tag3) + + post5 = post_factory(id=5) + + db.session.add_all( + [ + tag1, + tag2, + tag3, + post1, + post2, + post3, + post4, + post5, + ] + ) + db.session.flush() + verify_unpaged(input, expected_post_ids) diff --git a/server/szurubooru/tests/search/configs/test_tag_search_config.py b/server/szurubooru/tests/search/configs/test_tag_search_config.py index 8175b73..9fe9a80 100644 --- a/server/szurubooru/tests/search/configs/test_tag_search_config.py +++ b/server/szurubooru/tests/search/configs/test_tag_search_config.py @@ -134,8 +134,6 @@ def test_escaping(executor, tag_factory, input, expected_tag_names, db_driver): ) db.session.flush() - if db_driver and db.session.get_bind().driver != db_driver: - pytest.xfail() if expected_tag_names is None: with pytest.raises(errors.SearchError): executor.execute(input, offset=0, limit=100) |