Merge branch 'master' into hunternif

24 files changed, 370 insertions, 79 deletions

diff --git a/server/Dockerfile b/server/Dockerfile
index 9a597a1..4beec1c 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -1,4 +1,4 @@
-ARG ALPINE_VERSION=3.12
+ARG ALPINE_VERSION=3.13
 
 
 FROM alpine:$ALPINE_VERSION as prereqs
@@ -6,6 +6,7 @@ WORKDIR /opt/app
 
 RUN apk --no-cache add \
         python3 \
+        python3-dev \
         ffmpeg \
         py3-pip \
         # from requirements.txt:
@@ -18,10 +19,18 @@ RUN apk --no-cache add \
         py3-pynacl \
         py3-tz \
         py3-pyrfc3339 \
+        build-base \
+ && apk --no-cache add \
+        libheif \
+        libavif \
+        libheif-dev \
+        libavif-dev \
  && pip3 install --no-cache-dir --disable-pip-version-check \
         alembic \
         "coloredlogs==5.0" \
-        youtube-dl \
+        youtube_dl \
+        pillow-avif-plugin \
+        pyheif-pillow-opener \
  && apk --no-cache del py3-pip
 
 COPY ./ /opt/app/
diff --git a/server/requirements.txt b/server/requirements.txt
index d80ec06..2a09b24 100644
--- a/server/requirements.txt
+++ b/server/requirements.txt
@@ -1,7 +1,7 @@
 alembic>=0.8.5
 pyyaml>=3.11
 psycopg2-binary>=2.6.1
-SQLAlchemy>=1.0.12
+SQLAlchemy>=1.0.12, <1.4
 coloredlogs==5.0
 certifi>=2017.11.5
 numpy>=1.8.2
@@ -9,4 +9,6 @@ pillow>=4.3.0
 pynacl>=1.2.1
 pytz>=2018.3
 pyRFC3339>=1.0
-youtube-dl
+pillow-avif-plugin>=1.1.0
+pyheif-pillow-opener>=0.1.0
+youtube_dl
diff --git a/server/szurubooru/facade.py b/server/szurubooru/facade.py
index ecf34c7..a7e4844 100644
--- a/server/szurubooru/facade.py
+++ b/server/szurubooru/facade.py
@@ -10,7 +10,10 @@ import sqlalchemy.orm.exc
 
 from szurubooru import api, config, db, errors, middleware, rest
 from szurubooru.func.file_uploads import purge_old_uploads
-from szurubooru.func.posts import update_all_post_signatures
+from szurubooru.func.posts import (
+    update_all_md5_checksums,
+    update_all_post_signatures,
+)
 
 
 def _map_error(
@@ -125,6 +128,12 @@ def purge_old_uploads_daemon() -> None:
         time.sleep(60 * 5)
 
 
+_live_migrations = (
+    update_all_post_signatures,
+    update_all_md5_checksums,
+)
+
+
 def create_app() -> Callable[[Any, Any], Any]:
     """ Create a WSGI compatible App object. """
     validate_config()
@@ -134,13 +143,10 @@ def create_app() -> Callable[[Any, Any], Any]:
     if config.config["show_sql"]:
         logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
 
-    purge_thread = threading.Thread(target=purge_old_uploads_daemon)
-    purge_thread.daemon = True
-    purge_thread.start()
+    threading.Thread(target=purge_old_uploads_daemon, daemon=True).start()
 
-    hashing_thread = threading.Thread(target=update_all_post_signatures)
-    hashing_thread.daemon = False
-    hashing_thread.start()
+    for migration in _live_migrations:
+        threading.Thread(target=migration, daemon=False).start()
 
     db.session.commit()
 
diff --git a/server/szurubooru/func/image_hash.py b/server/szurubooru/func/image_hash.py
index fc7d141..a445e62 100644
--- a/server/szurubooru/func/image_hash.py
+++ b/server/szurubooru/func/image_hash.py
@@ -6,6 +6,10 @@ from typing import Any, Callable, List, Optional, Set, Tuple
 
 import numpy as np
 from PIL import Image
+import pillow_avif
+import pyheif
+from pyheif_pillow_opener import register_heif_opener
+register_heif_opener()
 
 from szurubooru import config, errors
 
@@ -40,7 +44,7 @@ def _preprocess_image(content: bytes) -> NpMatrix:
     try:
         img = Image.open(BytesIO(content))
         return np.asarray(img.convert("L"), dtype=np.uint8)
-    except IOError:
+    except (IOError, ValueError):
         raise errors.ProcessingError(
             "Unable to generate a signature hash " "for this image."
         )
diff --git a/server/szurubooru/func/images.py b/server/szurubooru/func/images.py
index 6413ac8..101bba8 100644
--- a/server/szurubooru/func/images.py
+++ b/server/szurubooru/func/images.py
@@ -4,7 +4,9 @@ import math
 import re
 import shlex
 import subprocess
+from io import BytesIO
 from typing import List
+from PIL import Image as PILImage
 
 from szurubooru import errors
 from szurubooru.func import mime, util
@@ -12,6 +14,13 @@ from szurubooru.func import mime, util
 logger = logging.getLogger(__name__)
 
 
+def convert_heif_to_png(content: bytes) -> bytes:
+    img = PILImage.open(BytesIO(content))
+    img_byte_arr = BytesIO()
+    img.save(img_byte_arr, format='PNG')
+    return img_byte_arr.getvalue()
+
+
 class Image:
     def __init__(self, content: bytes) -> None:
         self.content = content
@@ -252,7 +261,12 @@ class Image:
         ignore_error_if_data: bool = False,
         get_logs: bool = False,
     ) -> bytes:
-        extension = mime.get_extension(mime.get_mime_type(self.content))
+        mime_type = mime.get_mime_type(self.content)
+        if mime.is_heif(mime_type):
+            # FFmpeg does not support HEIF.
+            # https://trac.ffmpeg.org/ticket/6521
+            self.content = convert_heif_to_png(self.content)
+        extension = mime.get_extension(mime_type)
         assert extension
         with util.create_temp_file(suffix="." + extension) as handle:
             handle.write(self.content)
diff --git a/server/szurubooru/func/mailer.py b/server/szurubooru/func/mailer.py
index c4cf9db..0fe35ac 100644
--- a/server/szurubooru/func/mailer.py
+++ b/server/szurubooru/func/mailer.py
@@ -13,6 +13,10 @@ def send_mail(sender: str, recipient: str, subject: str, body: str) -> None:
     smtp = smtplib.SMTP(
         config.config["smtp"]["host"], int(config.config["smtp"]["port"])
     )
+    try:
+        smtp.starttls()
+    except smtplib.SMTPNotSupportedError:
+        pass
     smtp.login(config.config["smtp"]["user"], config.config["smtp"]["pass"])
     smtp.send_message(msg)
     smtp.quit()
diff --git a/server/szurubooru/func/mime.py b/server/szurubooru/func/mime.py
index 5f6279b..93c096b 100644
--- a/server/szurubooru/func/mime.py
+++ b/server/szurubooru/func/mime.py
@@ -21,10 +21,22 @@ def get_mime_type(content: bytes) -> str:
     if content[8:12] == b"WEBP":
         return "image/webp"
 
+    if content[0:2] == b"BM":
+        return "image/bmp"
+
+    if content[4:12] in (b"ftypavif", b"ftypavis"):
+        return "image/avif"
+
+    if content[4:12] == b"ftypmif1":
+        return "image/heif"
+
+    if content[4:12] in (b"ftypheic", b"ftypheix"):
+        return "image/heic"
+
     if content[0:4] == b"\x1A\x45\xDF\xA3":
         return "video/webm"
 
-    if content[4:12] in (b"ftypisom", b"ftypiso5", b"ftypmp42"):
+    if content[4:12] in (b"ftypisom", b"ftypiso5", b"ftypmp42", b"ftypM4V "):
         return "video/mp4"
 
     return "application/octet-stream"
@@ -37,6 +49,10 @@ def get_extension(mime_type: str) -> Optional[str]:
         "image/jpeg": "jpg",
         "image/png": "png",
         "image/webp": "webp",
+        "image/bmp": "bmp",
+        "image/avif": "avif",
+        "image/heif": "heif",
+        "image/heic": "heic",
         "video/mp4": "mp4",
         "video/webm": "webm",
         "application/octet-stream": "dat",
@@ -58,6 +74,10 @@ def is_image(mime_type: str) -> bool:
         "image/png",
         "image/gif",
         "image/webp",
+        "image/bmp",
+        "image/avif",
+        "image/heif",
+        "image/heic",
     )
 
 
@@ -67,3 +87,10 @@ def is_animated_gif(content: bytes) -> bool:
         get_mime_type(content) == "image/gif"
         and len(re.findall(pattern, content)) > 1
     )
+
+def is_heif(mime_type: str) -> bool:
+    return mime_type.lower() in (
+        "image/heif",
+        "image/heic",
+        "image/avif",
+    )
diff --git a/server/szurubooru/func/net.py b/server/szurubooru/func/net.py
index 4e4c222..9dff3c4 100644
--- a/server/szurubooru/func/net.py
+++ b/server/szurubooru/func/net.py
@@ -1,81 +1,89 @@
 import json
 import logging
-import os
+import subprocess
 import urllib.error
 import urllib.request
-from tempfile import NamedTemporaryFile
 from threading import Thread
 from typing import Any, Dict, List
 
-from youtube_dl import YoutubeDL
-from youtube_dl.utils import YoutubeDLError
-
 from szurubooru import config, errors
-from szurubooru.func import mime, util
+from szurubooru.func import mime
 
 logger = logging.getLogger(__name__)
+_dl_chunk_size = 2 ** 15
+
+
+class DownloadError(errors.ProcessingError):
+    pass
+
+
+class DownloadTooLargeError(DownloadError):
+    pass
 
 
 def download(url: str, use_video_downloader: bool = False) -> bytes:
     assert url
+    youtube_dl_error = None
+    if use_video_downloader:
+        try:
+            url = _get_youtube_dl_content_url(url) or url
+        except errors.ThirdPartyError as ex:
+            youtube_dl_error = ex
+
     request = urllib.request.Request(url)
     if config.config["user_agent"]:
         request.add_header("User-Agent", config.config["user_agent"])
     request.add_header("Referer", url)
+
+    content_buffer = b""
+    length_tally = 0
     try:
         with urllib.request.urlopen(request) as handle:
-            content = handle.read()
-    except Exception as ex:
-        raise errors.ProcessingError("Error downloading %s (%s)" % (url, ex))
+            while (chunk := handle.read(_dl_chunk_size)) :
+                length_tally += len(chunk)
+                if length_tally > config.config["max_dl_filesize"]:
+                    raise DownloadTooLargeError(url)
+                content_buffer += chunk
+    except urllib.error.HTTPError as ex:
+        raise DownloadError(url) from ex
+
     if (
-        use_video_downloader
-        and mime.get_mime_type(content) == "application/octet-stream"
+        youtube_dl_error
+        and mime.get_mime_type(content_buffer) == "application/octet-stream"
     ):
-        return _youtube_dl_wrapper(url)
-    return content
+        raise youtube_dl_error
+
+    return content_buffer
 
 
-def _youtube_dl_wrapper(url: str) -> bytes:
-    outpath = os.path.join(
-        config.config["data_dir"],
-        "temporary-uploads",
-        "youtubedl-" + util.get_sha1(url)[0:8] + ".dat",
-    )
-    options = {
-        "ignoreerrors": False,
-        "format": "best[ext=webm]/best[ext=mp4]/best[ext=flv]",
-        "logger": logger,
-        "max_filesize": config.config["max_dl_filesize"],
-        "max_downloads": 1,
-        "outtmpl": outpath,
-    }
+def _get_youtube_dl_content_url(url: str) -> str:
+    cmd = ["youtube-dl", "--format", "best", "--no-playlist"]
+    if config.config["user_agent"]:
+        cmd.extend(["--user-agent", config.config["user_agent"]])
+    cmd.extend(["--get-url", url])
     try:
-        with YoutubeDL(options) as ydl:
-            ydl.extract_info(url, download=True)
-        with open(outpath, "rb") as f:
-            return f.read()
-    except YoutubeDLError as ex:
-        raise errors.ThirdPartyError(
-            "Error downloading video %s (%s)" % (url, ex)
+        return (
+            subprocess.run(cmd, text=True, capture_output=True, check=True)
+            .stdout.split("\n")[0]
+            .strip()
         )
-    except FileNotFoundError:
+    except subprocess.CalledProcessError:
         raise errors.ThirdPartyError(
-            "Error downloading video %s (file could not be saved)" % (url)
-        )
+            "Could not extract content location from %s" % (url)
+        ) from None
 
 
 def post_to_webhooks(payload: Dict[str, Any]) -> List[Thread]:
     threads = [
-        Thread(target=_post_to_webhook, args=(webhook, payload))
+        Thread(target=_post_to_webhook, args=(webhook, payload), daemon=False)
         for webhook in (config.config["webhooks"] or [])
     ]
     for thread in threads:
-        thread.daemon = False
         thread.start()
     return threads
 
 
-def _post_to_webhook(webhook: str, payload: Dict[str, Any]) -> None:
+def _post_to_webhook(webhook: str, payload: Dict[str, Any]) -> int:
     req = urllib.request.Request(webhook)
     req.data = json.dumps(
         payload,
@@ -89,6 +97,6 @@ def _post_to_webhook(webhook: str, payload: Dict[str, Any]) -> None:
                 f"Webhook {webhook} returned {res.status} {res.reason}"
             )
         return res.status
-    except urllib.error.URLError as e:
-        logger.warning(f"Unable to call webhook {webhook}: {str(e)}")
+    except urllib.error.URLError as ex:
+        logger.warning(f"Unable to call webhook {webhook}: {ex}")
         return 400
diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py
index ee7c31a..107fc5d 100644
--- a/server/szurubooru/func/posts.py
+++ b/server/szurubooru/func/posts.py
@@ -175,6 +175,7 @@ class PostSerializer(serialization.BaseSerializer):
             "type": self.serialize_type,
             "mimeType": self.serialize_mime,
             "checksum": self.serialize_checksum,
+            "checksumMD5": self.serialize_checksum_md5,
             "fileSize": self.serialize_file_size,
             "canvasWidth": self.serialize_canvas_width,
             "canvasHeight": self.serialize_canvas_height,
@@ -230,6 +231,9 @@ class PostSerializer(serialization.BaseSerializer):
     def serialize_checksum(self) -> Any:
         return self.post.checksum
 
+    def serialize_checksum_md5(self) -> Any:
+        return self.post.checksum_md5
+
     def serialize_file_size(self) -> Any:
         return self.post.file_size
 
@@ -602,7 +606,25 @@ def update_all_post_signatures() -> None:
                 post, files.get(get_post_content_path(post))
             )
             db.session.commit()
-            logger.info("Hashed Post %d", post.post_id)
+            logger.info("Created Signature - Post %d", post.post_id)
+        except Exception as ex:
+            logger.exception(ex)
+
+
+def update_all_md5_checksums() -> None:
+    posts_to_hash = (
+        db.session.query(model.Post)
+        .filter(model.Post.checksum_md5 == None)  # noqa: E711
+        .order_by(model.Post.post_id.asc())
+        .all()
+    )
+    for post in posts_to_hash:
+        try:
+            post.checksum_md5 = util.get_md5(
+                files.get(get_post_content_path(post))
+            )
+            db.session.commit()
+            logger.info("Created MD5 - Post %d", post.post_id)
         except Exception as ex:
             logger.exception(ex)
 
@@ -630,6 +652,7 @@ def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
         )
 
     post.checksum = util.get_sha1(content)
+    post.checksum_md5 = util.get_md5(content)
     other_post = (
         db.session.query(model.Post)
         .filter(model.Post.checksum == post.checksum)
@@ -652,7 +675,8 @@ def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
         image = images.Image(content)
         post.canvas_width = image.width
         post.canvas_height = image.height
-    except errors.ProcessingError:
+    except errors.ProcessingError as ex:
+        logger.exception(ex)
         if not config.config["allow_broken_uploads"]:
             raise InvalidPostContentError("Unable to process image metadata")
         else:
diff --git a/server/szurubooru/migrations/versions/adcd63ff76a2_add_md5_checksums_to_posts.py b/server/szurubooru/migrations/versions/adcd63ff76a2_add_md5_checksums_to_posts.py
new file mode 100644
index 0000000..4a1b202
--- /dev/null
+++ b/server/szurubooru/migrations/versions/adcd63ff76a2_add_md5_checksums_to_posts.py
@@ -0,0 +1,22 @@
+"""
+Add MD5 checksums to posts
+
+Revision ID: adcd63ff76a2
+Created at: 2021-01-05 17:08:21.741601
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "adcd63ff76a2"
+down_revision = "c867abb456b1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("post", sa.Column("checksum_md5", sa.Unicode(32)))
+
+
+def downgrade():
+    op.drop_column("post", "checksum_md5")
diff --git a/server/szurubooru/model/post.py b/server/szurubooru/model/post.py
index cf68860..deb3cc8 100644
--- a/server/szurubooru/model/post.py
+++ b/server/szurubooru/model/post.py
@@ -217,6 +217,7 @@ class Post(Base):
     # content description
     type = sa.Column("type", sa.Unicode(32), nullable=False)
     checksum = sa.Column("checksum", sa.Unicode(64), nullable=False)
+    checksum_md5 = sa.Column("checksum_md5", sa.Unicode(32))
     file_size = sa.Column("file_size", sa.BigInteger)
     canvas_width = sa.Column("image_width", sa.Integer)
     canvas_height = sa.Column("image_height", sa.Integer)
diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py
index 71bd16a..eda4084 100644
--- a/server/szurubooru/search/configs/post_search_config.py
+++ b/server/szurubooru/search/configs/post_search_config.py
@@ -339,10 +339,14 @@ class PostSearchConfig(BaseSearchConfig):
                     ),
                 ),
                 (
-                    ["content-checksum"],
+                    ["content-checksum", "sha1"],
                     search_util.create_str_filter(model.Post.checksum),
                 ),
                 (
+                    ["md5"],
+                    search_util.create_str_filter(model.Post.checksum_md5),
+                ),
+                (
                     ["file-size"],
                     search_util.create_num_filter(model.Post.file_size),
                 ),
diff --git a/server/szurubooru/tests/assets/avif-avis.avif b/server/szurubooru/tests/assets/avif-avis.avif
new file mode 100644
index 0000000..3b27f29
--- /dev/null
+++ b/server/szurubooru/tests/assets/avif-avis.avif
diff --git a/server/szurubooru/tests/assets/avif-similar.avif b/server/szurubooru/tests/assets/avif-similar.avif
new file mode 100644
index 0000000..476f3f5
--- /dev/null
+++ b/server/szurubooru/tests/assets/avif-similar.avif
diff --git a/server/szurubooru/tests/assets/avif.avif b/server/szurubooru/tests/assets/avif.avif
new file mode 100644
index 0000000..edf4960
--- /dev/null
+++ b/server/szurubooru/tests/assets/avif.avif
diff --git a/server/szurubooru/tests/assets/bmp.bmp b/server/szurubooru/tests/assets/bmp.bmp
new file mode 100644
index 0000000..3a18269
--- /dev/null
+++ b/server/szurubooru/tests/assets/bmp.bmp
diff --git a/server/szurubooru/tests/assets/heic-heix.heic b/server/szurubooru/tests/assets/heic-heix.heic
new file mode 100644
index 0000000..27d1268
--- /dev/null
+++ b/server/szurubooru/tests/assets/heic-heix.heic
diff --git a/server/szurubooru/tests/assets/heic.heic b/server/szurubooru/tests/assets/heic.heic
new file mode 100644
index 0000000..f1d2ed0
--- /dev/null
+++ b/server/szurubooru/tests/assets/heic.heic
diff --git a/server/szurubooru/tests/assets/heif-similar.heif b/server/szurubooru/tests/assets/heif-similar.heif
new file mode 100644
index 0000000..54385f2
--- /dev/null
+++ b/server/szurubooru/tests/assets/heif-similar.heif
diff --git a/server/szurubooru/tests/assets/heif.heif b/server/szurubooru/tests/assets/heif.heif
new file mode 100644
index 0000000..7fec72b
--- /dev/null
+++ b/server/szurubooru/tests/assets/heif.heif
diff --git a/server/szurubooru/tests/func/test_image_hash.py b/server/szurubooru/tests/func/test_image_hash.py
index e7028b6..5a5dc71 100644
--- a/server/szurubooru/tests/func/test_image_hash.py
+++ b/server/szurubooru/tests/func/test_image_hash.py
@@ -27,3 +27,53 @@ def test_signature_functions(read_asset, config_injector):
     words2 = image_hash.generate_words(sig2)
     words_match = sum(word1 == word2 for word1, word2 in zip(words1, words2))
     assert words_match == 18
+
+
+def test_signature_heif(read_asset, config_injector):
+    sig1 = image_hash.generate_signature(read_asset("heif.heif"))
+    sig2 = image_hash.generate_signature(read_asset("heif-similar.heif"))
+
+    sig1_repacked = image_hash.unpack_signature(
+        image_hash.pack_signature(sig1)
+    )
+    sig2_repacked = image_hash.unpack_signature(
+        image_hash.pack_signature(sig2)
+    )
+    assert array_equal(sig1, sig1_repacked)
+    assert array_equal(sig2, sig2_repacked)
+
+    dist1 = image_hash.normalized_distance([sig1], sig2)
+    assert abs(dist1[0] - 0.136777724290135) < 1e-8
+
+    dist2 = image_hash.normalized_distance([sig2], sig2)
+    assert abs(dist2[0]) < 1e-8
+
+    words1 = image_hash.generate_words(sig1)
+    words2 = image_hash.generate_words(sig2)
+    words_match = sum(word1 == word2 for word1, word2 in zip(words1, words2))
+    assert words_match == 43
+
+
+def test_signature_avif(read_asset, config_injector):
+    sig1 = image_hash.generate_signature(read_asset("avif.avif"))
+    sig2 = image_hash.generate_signature(read_asset("avif-similar.avif"))
+
+    sig1_repacked = image_hash.unpack_signature(
+        image_hash.pack_signature(sig1)
+    )
+    sig2_repacked = image_hash.unpack_signature(
+        image_hash.pack_signature(sig2)
+    )
+    assert array_equal(sig1, sig1_repacked)
+    assert array_equal(sig2, sig2_repacked)
+
+    dist1 = image_hash.normalized_distance([sig1], sig2)
+    assert abs(dist1[0] - 0.22628712858355998) < 1e-8
+
+    dist2 = image_hash.normalized_distance([sig2], sig2)
+    assert abs(dist2[0]) < 1e-8
+
+    words1 = image_hash.generate_words(sig1)
+    words2 = image_hash.generate_words(sig2)
+    words_match = sum(word1 == word2 for word1, word2 in zip(words1, words2))
+    assert words_match == 12
diff --git a/server/szurubooru/tests/func/test_mime.py b/server/szurubooru/tests/func/test_mime.py
index 0d8f645..b33746b 100644
--- a/server/szurubooru/tests/func/test_mime.py
+++ b/server/szurubooru/tests/func/test_mime.py
@@ -13,6 +13,12 @@ from szurubooru.func import mime
         ("jpeg.jpg", "image/jpeg"),
         ("gif.gif", "image/gif"),
         ("webp.webp", "image/webp"),
+        ("bmp.bmp", "image/bmp"),
+        ("avif.avif", "image/avif"),
+        ("avif-avis.avif", "image/avif"),
+        ("heif.heif", "image/heif"),
+        ("heic.heic", "image/heic"),
+        ("heic-heix.heic", "image/heic"),
         ("text.txt", "application/octet-stream"),
     ],
 )
@@ -34,6 +40,10 @@ def test_get_mime_type_for_empty_file():
         ("image/jpeg", "jpg"),
         ("image/gif", "gif"),
         ("image/webp", "webp"),
+        ("image/bmp", "bmp"),
+        ("image/avif", "avif"),
+        ("image/heif", "heif"),
+        ("image/heic", "heic"),
         ("application/octet-stream", "dat"),
     ],
 )
@@ -75,9 +85,17 @@ def test_is_video(input_mime_type, expected_state):
         ("image/gif", True),
         ("image/png", True),
         ("image/jpeg", True),
+        ("image/bmp", True),
+        ("image/avif", True),
+        ("image/heic", True),
+        ("image/heif", True),
         ("IMAGE/GIF", True),
         ("IMAGE/PNG", True),
         ("IMAGE/JPEG", True),
+        ("IMAGE/BMP", True),
+        ("IMAGE/AVIF", True),
+        ("IMAGE/HEIC", True),
+        ("IMAGE/HEIF", True),
         ("image/anything_else", False),
         ("not an image", False),
     ],
@@ -95,3 +113,26 @@ def test_is_image(input_mime_type, expected_state):
 )
 def test_is_animated_gif(read_asset, input_path, expected_state):
     assert mime.is_animated_gif(read_asset(input_path)) == expected_state
+
+
+@pytest.mark.parametrize(
+    "input_mime_type,expected_state",
+    [
+        ("image/gif", False),
+        ("image/png", False),
+        ("image/jpeg", False),
+        ("image/avif", True),
+        ("image/heic", True),
+        ("image/heif", True),
+        ("IMAGE/GIF", False),
+        ("IMAGE/PNG", False),
+        ("IMAGE/JPEG", False),
+        ("IMAGE/AVIF", True),
+        ("IMAGE/HEIC", True),
+        ("IMAGE/HEIF", True),
+        ("image/anything_else", False),
+        ("not an image", False),
+    ],
+)
+def test_is_heif(input_mime_type, expected_state):
+    assert mime.is_heif(input_mime_type) == expected_state
diff --git a/server/szurubooru/tests/func/test_net.py b/server/szurubooru/tests/func/test_net.py
index 65e9048..c5b4c73 100644
--- a/server/szurubooru/tests/func/test_net.py
+++ b/server/szurubooru/tests/func/test_net.py
@@ -1,6 +1,3 @@
-from datetime import datetime
-from unittest.mock import patch
-
 import pytest
 
 from szurubooru import errors
@@ -69,40 +66,38 @@ def test_download():
     "url",
     [
         "https://samples.ffmpeg.org/MPEG-4/video.mp4",
+        "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
     ],
 )
 def test_too_large_download(url):
-    pytest.xfail("Download limit not implemented yet")
-    with pytest.raises(errors.ProcessingError):
-        net.download(url)
+    with pytest.raises(net.DownloadTooLargeError):
+        net.download(url, use_video_downloader=True)
 
 
 @pytest.mark.parametrize(
     "url,expected_sha1",
     [
         (
-            "https://www.youtube.com/watch?v=C0DPdy98e4c",
-            "365af1c8f59c6865e1a84c6e13e3e25ff89e0ba1",
+            "https://gfycat.com/immaterialchillyiberianmole",
+            "0125976d2439e651b6863438db30de58f79f7754",
+        ),
+        (
+            "https://upload.wikimedia.org/wikipedia/commons/a/ad/Utah_teapot.png",  # noqa: E501
+            "cfadcbdeda1204dc1363ee5c1969191f26be2e41",
         ),
         (
-            "https://gfycat.com/immaterialchillyiberianmole",
-            "953000e81d7bd1da95ce264f872e7b6c4a6484be",
+            "https://i.imgur.com/GPgh0AN.jpg",
+            "26861a4663fedae48e5beed3eec5156ded20640f",
         ),
     ],
 )
-def test_video_download(url, expected_sha1):
+def test_content_download(url, expected_sha1):
     actual_content = net.download(url, use_video_downloader=True)
     assert get_sha1(actual_content) == expected_sha1
 
 
-@pytest.mark.parametrize(
-    "url",
-    [
-        "https://samples.ffmpeg.org/flac/short.flac",  # not a video
-        "https://www.youtube.com/watch?v=dQw4w9WgXcQ",  # video too large
-    ],
-)
-def test_failed_video_download(url):
+def test_bad_content_downlaod():
+    url = "http://info.cern.ch/hypertext/WWW/TheProject.html"
     with pytest.raises(errors.ThirdPartyError):
         net.download(url, use_video_downloader=True)
 
diff --git a/server/szurubooru/tests/func/test_posts.py b/server/szurubooru/tests/func/test_posts.py
index af6121a..e1be764 100644
--- a/server/szurubooru/tests/func/test_posts.py
+++ b/server/szurubooru/tests/func/test_posts.py
@@ -147,6 +147,7 @@ def test_serialize_post(
         post.source = "4gag"
         post.type = model.Post.TYPE_IMAGE
         post.checksum = "deadbeef"
+        post.checksum_md5 = "deadbeef"
         post.mime_type = "image/jpeg"
         post.file_size = 100
         post.user = user_factory(name="post author")
@@ -231,6 +232,7 @@ def test_serialize_post(
             "source": "4gag",
             "type": "image",
             "checksum": "deadbeef",
+            "checksumMD5": "deadbeef",
             "fileSize": 100,
             "canvasWidth": 200,
             "canvasHeight": 300,
@@ -424,6 +426,48 @@ def test_update_post_source_with_too_long_string():
         ),
         (
             False,
+            "bmp.bmp",
+            "image/bmp",
+            model.Post.TYPE_IMAGE,
+            "1_244c8840887984c4.bmp",
+        ),
+        (
+            False,
+            "avif.avif",
+            "image/avif",
+            model.Post.TYPE_IMAGE,
+            "1_244c8840887984c4.avif",
+        ),
+        (
+            False,
+            "avif-avis.avif",
+            "image/avif",
+            model.Post.TYPE_IMAGE,
+            "1_244c8840887984c4.avif",
+        ),
+        (
+            False,
+            "heic.heic",
+            "image/heic",
+            model.Post.TYPE_IMAGE,
+            "1_244c8840887984c4.heic",
+        ),
+        (
+            False,
+            "heic-heix.heic",
+            "image/heic",
+            model.Post.TYPE_IMAGE,
+            "1_244c8840887984c4.heic",
+        ),
+        (
+            False,
+            "heif.heif",
+            "image/heif",
+            model.Post.TYPE_IMAGE,
+            "1_244c8840887984c4.heif",
+        ),
+        (
+            False,
             "gif-animated.gif",
             "image/gif",
             model.Post.TYPE_ANIMATION,
@@ -463,8 +507,11 @@ def test_update_post_content_for_new_post(
     expected_type,
     output_file_name,
 ):
-    with patch("szurubooru.func.util.get_sha1"):
+    with patch("szurubooru.func.util.get_sha1"), patch(
+        "szurubooru.func.util.get_md5"
+    ):
         util.get_sha1.return_value = "crc"
+        util.get_md5.return_value = "md5"
         config_injector(
             {
                 "data_dir": str(tmpdir.mkdir("data")),
@@ -490,6 +537,7 @@ def test_update_post_content_for_new_post(
         assert post.mime_type == expected_mime_type
         assert post.type == expected_type
         assert post.checksum == "crc"
+        assert post.checksum_md5 == "md5"
         assert os.path.exists(output_file_path)
         if post.type in (model.Post.TYPE_IMAGE, model.Post.TYPE_ANIMATION):
             assert db.session.query(model.PostSignature).count() == 1
@@ -725,6 +773,38 @@ def test_update_post_content_leaving_custom_thumbnail(
     assert os.path.exists(generated_path)
 
 
+@pytest.mark.parametrize("filename", ("avif.avif", "heic.heic", "heif.heif"))
+def test_update_post_content_convert_heif_to_png_when_processing(
+    tmpdir, config_injector, read_asset, post_factory, filename
+):
+    config_injector(
+        {
+            "data_dir": str(tmpdir.mkdir("data")),
+            "thumbnails": {
+                "post_width": 300,
+                "post_height": 300,
+            },
+            "secret": "test",
+            "allow_broken_uploads": False,
+        }
+    )
+    post = post_factory(id=1)
+    db.session.add(post)
+    posts.update_post_content(post, read_asset(filename))
+    posts.update_post_thumbnail(post, read_asset(filename))
+    db.session.flush()
+    generated_path = (
+        "{}/data/generated-thumbnails/".format(tmpdir)
+        + "1_244c8840887984c4.jpg"
+    )
+    source_path = (
+        "{}/data/posts/custom-thumbnails/".format(tmpdir)
+        + "1_244c8840887984c4.dat"
+    )
+    assert os.path.exists(source_path)
+    assert os.path.exists(generated_path)
+
+
 def test_update_post_tags(tag_factory):
     post = model.Post()
     with patch("szurubooru.func.tags.get_or_create_tags_by_names"):