From da213ea193973e079e68759da9e835202ed04aea Mon Sep 17 00:00:00 2001 From: "Jakob L. Kreuze" Date: Sun, 23 Jun 2024 20:00:57 -0400 Subject: Initial commit --- lib/newgrounds/__init__.py | 1 + lib/newgrounds/newgrounds.py | 567 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 568 insertions(+) create mode 100644 lib/newgrounds/__init__.py create mode 100644 lib/newgrounds/newgrounds.py (limited to 'lib/newgrounds') diff --git a/lib/newgrounds/__init__.py b/lib/newgrounds/__init__.py new file mode 100644 index 0000000..d48ce89 --- /dev/null +++ b/lib/newgrounds/__init__.py @@ -0,0 +1 @@ +from .newgrounds import * diff --git a/lib/newgrounds/newgrounds.py b/lib/newgrounds/newgrounds.py new file mode 100644 index 0000000..10a5b57 --- /dev/null +++ b/lib/newgrounds/newgrounds.py @@ -0,0 +1,567 @@ +# Copyright © 2024 Jakob L. Kreuze +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Video plugin for animations posted to Newgrounds. +""" + +import concurrent.futures +from datetime import datetime +from dataclasses import dataclass, field +from enum import Enum + +from dateutil import parser as dateutil_parser +import json +import os +import sys +from typing import List +from urllib.parse import urlencode, parse_qsl + +from bs4 import BeautifulSoup +import requests + +Suitability = Enum("Suitability", ["E", "T", "M", "A"]) +Conduct = Enum( + "Conduct", + ["MOVIES", "AUDIO", "ART", "ALBUMS", "PLAYLISTS", "COLLECTIONS", "SERIES"], +) +Grouping = Enum("Grouping", ["FEATURED", "LATEST", "POPULAR"]) +SortBy = Enum( + "SortBy", + [ + "RELEVANCE", + "DATE_ASC", + "DATE_DESC", + "SCORE_ASC", + "SCORE_DESC", + "VIEWS_ASC", + "VIEWS_DESC", + ], +) +Interval = Enum("Interval", ["TODAY", "YESTERDAY", "WEEK", "MONTH", "YEAR"]) +AudioArchetype = Enum("AudioArchetype", ["MUSIC", "VOICE", "PODCASTS"]) + + +def stringify_enum(enum_value): + return enum_value.name.lower().replace("_", "-") + + +@dataclass +class VisualLink: + title: str + thumbnail: str + author: str + url: str + + +@dataclass +class Movie: + title: str + thumbnail: str + content_url: str + url: str + views: int = 0 + faves: int = 0 + votes: int = 0 + score: float = 0.0 + upload_date: datetime = datetime.fromtimestamp(0) + genre: str = "Unknown" + tags: List[str] = field(default_factory=lambda: []) + description: str = "" + authors: List[str] = field(default_factory=lambda: []) + suitability: Suitability = Suitability.E + + def __post_init__(self): + pass + + def url(self): + pass + + +@dataclass +class Audio: + title: str + thumbnail: str + content_url: str + url: str + views: int = 0 + faves: int = 0 + votes: int = 0 + score: float = 0.0 + upload_date: datetime = datetime.fromtimestamp(0) + genre: str = "Unknown" + tags: list[str] = field(default_factory=lambda: []) + description: str = "" + authors: list[str] = field(default_factory=lambda: []) + suitability: Suitability = Suitability.E + + def __post_init__(self): + pass + + def url(self): + pass + + +MOVIE_CATEGORIES = { + "All": 0, + "Action": 45, + "Comedy - Original": 60, + "Comedy - Parody": 61, + "Drama": 47, + "Experimental": 49, + "Informative": 48, + "Music Video": 50, + "Other": 51, + "Spam": 55, +} + +AUDIO_CATEGORIES = { + "All": 0, + "Easy Listening - Classical": 3, + "Easy Listening - Jazz": 18, + "Easy Listening - Solo Instrument": 51, + "Electronic - Ambient": 5, + "Electronic - Chipstep": 48, + "Electronic - Dance": 6, + "Electronic - Drum N Bass": 7, + "Electronic - Dubstep": 41, + "Electronic - House": 9, + "Electronic - Industrial": 8, + "Electronic - New Wave": 20, + "Electronic - Synthwave": 57, + "Electronic - Techno": 10, + "Electronic - Trance": 11, + "Electronic - Video Game": 12, + "Hip Hop, Rap, R&B - Hip Hop - Modern": 17, + "Hip Hop, Rap, R&B - Hip Hop - Olskool": 16, + "Hip Hop, Rap, R&B - Nerdcore": 47, + "Hip Hop, Rap, R&B - R&B": 21, + "Metal, Rock - Brit Pop": 22, + "Metal, Rock - Classic Rock": 23, + "Metal, Rock - General Rock": 24, + "Metal, Rock - Grunge": 25, + "Metal, Rock - Heavy Metal": 15, + "Metal, Rock - Indie": 26, + "Metal, Rock - Pop": 27, + "Metal, Rock - Punk": 28, + "Other - Cinematic": 50, + "Other - Experimental": 49, + "Other - Funk": 13, + "Other - Fusion": 52, + "Other - Goth": 14, + "Other - Miscellaneous": 39, + "Other - Ska": 29, + "Other - World": 19, + "Southern Flavor - Bluegrass": 1, + "Southern Flavor - Blues": 2, + "Southern Flavor - Country": 4, +} + +ART_CATEGORIES = { + "All": 0, +} + +SESSION_COOKIE_NAME = "vmkIdu5l8m" + + +class Newgrounds(object): + def __init__(self): + pass + + def _request(self, method, url, **kwargs): + kwargs = dict(kwargs) + # TODO: Test if we need to open a new session? + if hasattr(self, "session") and self.session is not None: + if "cookies" in kwargs: + kwargs["cookies"][SESSION_COOKIE_NAME] = self.session + else: + kwargs["cookies"] = {SESSION_COOKIE_NAME: self.session} + r = requests.request(method, url, **kwargs) + return r + + def login(self, username, password): + self.session = None + html = self._request("GET", "https://www.newgrounds.com/login").text + soup = BeautifulSoup(html, "html.parser") + + form = soup.find("form", method="post") + endpoint = form["action"] + auth = form.find("input")["value"] + + r = self._request( + "POST", + "https://www.newgrounds.com" + endpoint, + data={ + "auth": auth, + "username": username, + "password": password, + "code": "", + "codehint": "------", + }, + cookies={ + "passport-auth": auth, + }, + ) + self.session = r.cookies[SESSION_COOKIE_NAME] + + def _test_logged_in(session): + return ( + requests.get( + "https://www.newgrounds.com/account", + cookies={SESSION_COOKIE_NAME: session}, + ).status_code + != 401 + ) + + def series(self, official=True, sort_by=SortBy.DATE_DESC, offset=1): + html = self._request( + "GET", + "https://www.newgrounds.com/series", + params={ + "filter": "all" if not official else None, + "order": stringify_enum(sort_by), + "page": offset, + }, + ).text + soup = BeautifulSoup(html, "html.parser") + visual_links = [ + json.loads(li["data-visual-link"]) + for li in soup.find_all("li", "visual-link-container") + ] + return self._fetch_visual_links(visual_links) + + def collections(self, official=True, sort_by=SortBy.DATE_DESC, offset=1): + html = self._request( + "GET", + "https://www.newgrounds.com/collections", + params={ + "filter": "all" if not official else None, + "order": stringify_enum(sort_by), + "page": offset, + }, + ).text + soup = BeautifulSoup(html, "html.parser") + visual_links = [ + json.loads(li["data-visual-link"]) + for li in soup.find_all("li", "visual-link-container") + ] + return self._fetch_visual_links(visual_links) + + def playlist_entries(self, url): + html = self._request("GET", url).text + soup = BeautifulSoup(html, "html.parser") + visual_links = [ + json.loads(li["data-visual-link"]) + for li in soup.find_all("li", "visual-link-container") + ] + urls = [card["url"] for card in self._fetch_visual_links(visual_links)] + return self._fetch_metadata_batch(urls) + + def _fetch_metadata_batch(self, urls): + result = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + # Start the load operations and mark each future with its URL + future_to_url = { + executor.submit(self._fetch_metadata, url): url for url in urls + } + for future in concurrent.futures.as_completed(future_to_url): + url = future_to_url[future] + result.append(future.result()) + return result + + def _fetch_visual_links(self, visual_links): + r = self._request( + "POST", + "https://www.newgrounds.com/visual-links-fetch", + params={ + "X-Requested-With": "XMLHttpRequest", + }, + data={ + "ids": json.dumps(visual_links).replace(" ", ""), + "component_params[include_author]": "1", + "include_all_suitabilities": "0", + "isAjaxRequest": "1", + }, + ) + result = r.json() + entries = [] + if isinstance(result["partials"], dict): + for partial in result["partials"].values(): + if isinstance(partial, dict): + for partial in partial.values(): + soup = BeautifulSoup(partial, "html.parser") + title = soup.find("h4").string + thumbnail = soup.find("img")["src"] + author = soup.find("strong").string + url = soup.find("a")["href"] + entries.append(VisualLink(title, thumbnail, author, url)) + return entries + + def _fetch_movie_content_url(self, movie_id): + response = self._request( + "GET", + "https://www.newgrounds.com/portal/video/{}".format(movie_id), + headers={"X-Requested-With": "XMLHttpRequest"}, + ).json() + preferences = ["1080p", "720p", "360p"] + for preference in preferences: + if preference in response["sources"]: + source = next( + filter( + lambda x: x["type"] == "video/mp4", + response["sources"][preference], + ) + ) + if source is not None: + return source["src"] + + def _fetch_movie_metadata(self, url): + html = self._request("GET", url).text + soup = BeautifulSoup(html, "html.parser") + + title = soup.find("div", id="embed_header").h2.string + thumbnail = soup.find("meta", property="og:image") + if thumbnail is not None: + thumbnail = thumbnail["content"] + + movie_id = soup.find("meta", property="og:url")["content"].strip( + "https://www.newgrounds.com/portal/view/" + ) + content_url = self._fetch_movie_content_url(movie_id) + + metadata = { + "title": title, + "thumbnail": thumbnail, + "content_url": content_url, + "url": url, + } + + sidestats_popularity = soup.find_all("dl", "sidestats")[0].find_all("dd") + if len(sidestats_popularity) == 4: + metadata["views"] = int(sidestats_popularity[0].string.replace(",", "")) + faves = sidestats_popularity[1].a + if faves is not None: + metadata["faves"] = int(faves.string.replace(",", "")) + metadata["votes"] = int(sidestats_popularity[2].string.replace(",", "")) + score = sidestats_popularity[3].span + if score is not None: + metadata["score"] = float(score.string) + + sidestats_date_and_genre = soup.find_all("dl", "sidestats")[1].find_all("dd") + if len(sidestats_date_and_genre) == 3: + metadata["upload_date"] = dateutil_parser.parse( + sidestats_date_and_genre[0].string + + " " + + sidestats_date_and_genre[1].string + ) + genre = sidestats_date_and_genre[2].a + if genre is not None: + metadata["genre"] = genre.string + + tags = soup.find("dd", "tags") + if tags is not None: + metadata["tags"] = [tag.string for tag in tags.find_all("a")] + + author_comments = soup.find("div", id="author_comments") + metadata["description"] = " ".join([s for s in author_comments.strings]).strip() + + authors = soup.find("ul", "authorlinks") + metadata["authors"] = [ + author.string + for author in authors.find_all("a") + if author.get("class") is None + ] + + if len(soup.find_all("h2", "rated-a")) != 0: + rating = "a" + elif len(soup.find_all("h2", "rated-m")) != 0: + rating = "m" + elif len(soup.find_all("h2", "rated-t")) != 0: + rating = "t" + else: + rating = "e" + + return Movie(**metadata) + + def _fetch_audio_metadata(self, url): + html = self._request("GET", url).text + soup = BeautifulSoup(html, "html.parser") + + title = soup.find("div", "pod-head").h2.string + thumbnail = soup.find("meta", property="og:image") + if thumbnail is not None: + thumbnail = thumbnail["content"] + content_url = soup.find("meta", property="og:audio")["content"] + + metadata = { + "title": title, + "thumbnail": thumbnail, + "content_url": content_url, + "url": url, + } + + sidestats_popularity = soup.find_all("dl", "sidestats")[0].find_all("dd") + if len(sidestats_popularity) == 4: + metadata["views"] = int(sidestats_popularity[0].string.replace(",", "")) + faves = sidestats_popularity[1].a + if faves is not None: + metadata["faves"] = int(faves.string.replace(",", "")) + metadata["votes"] = int(sidestats_popularity[2].string.replace(",", "")) + score = sidestats_popularity[3].span + if score is not None: + metadata["score"] = float(score.string) + + sidestats_date_and_genre = soup.find_all("dl", "sidestats")[1].find_all("dd") + if len(sidestats_date_and_genre) >= 3: + metadata["upload_date"] = dateutil_parser.parse( + sidestats_date_and_genre[0].string + + " " + + sidestats_date_and_genre[1].string + ) + genre = sidestats_date_and_genre[2].a + if genre is not None: + metadata["genre"] = genre.string + + if len(sidestats_date_and_genre) >= 5: + metadata["length"] = sidestats_date_and_genre[5].string.strip() + + tags = soup.find("dd", "tags") + if tags is not None: + metadata["tags"] = [tag.string for tag in tags.find_all("a")] + + # author_comments = soup.find("div", id="author_comments") + # metadata["description"] = " ".join([s for s in author_comments.strings]).strip() + + authors = soup.find("div", "item-user") + metadata["author"] = [ + author.string + for author in authors.find_all("a") + if author.get("class") is None + ][0] + + if len(soup.find_all("h2", "rated-a")) != 0: + rating = "a" + elif len(soup.find_all("h2", "rated-m")) != 0: + rating = "m" + elif len(soup.find_all("h2", "rated-t")) != 0: + rating = "t" + else: + rating = "e" + + return Audio(**metadata) + + def _fetch_metadata(self, url): + if "newgrounds.com/audio/" in url: + return self._fetch_audio_metadata(url) + elif "newgrounds.com/art/" in url: + return self._fetch_art_metadata(url) + else: + return self._fetch_movie_metadata(url) + + def search(self, term, conduct=Conduct.MOVIES, sort_by=SortBy.RELEVANCE, offset=1): + """ + Return a list of URLs matching a given search term. + """ + response = self._request( + "GET", + "https://www.newgrounds.com/search/conduct/{}".format( + stringify_enum(conduct) + ), + params={ + "suitabilities": "etm", + "sort": stringify_enum(sort_by), + "terms": term, + "page": offset, + "inner": 1, + }, + headers={ + "X-Requested-With": "XMLHttpRequest", + }, + ).json() + + soup = BeautifulSoup(response["content"], "html.parser") + urls = [tag["href"] for tag in soup.find_all("a", "item-portalsubmission")] + return self._fetch_metadata_batch(urls) + + def front_page( + self, + conduct=Conduct.MOVIES, + grouping=Grouping.FEATURED, + interval=Interval.TODAY, + sort_by=SortBy.DATE_DESC, + category="All", + offset=0, + ): + # TODO: Support `after` parameter (e.g. `before=2024-06-04`) + # TODO: Support `before` parameter (e.g. `before=2024-06-04`) + URLS = { + Grouping.FEATURED.value: "featured", + Grouping.LATEST.value: "browse", + Grouping.POPULAR.value: "popular", + } + + assert conduct == Conduct.MOVIES or conduct == Conduct.AUDIO + assert isinstance(category, str) + assert isinstance(offset, int) + + if conduct == Conduct.MOVIES: + assert category in MOVIE_CATEGORIES + elif conduct == Conduct.AUDIO: + assert category in AUDIO_CATEGORIES + + url = URLS[grouping] + response = self._request( + "GET", + "https://www.newgrounds.com/movies/" + url + if conduct == Conduct.MOVIES + else "https://www.newgrounds.com/audio/" + url, + params={ + "interval": interval, + "sort": stringify_enum(sort_by), + "genre": category, + "offset": offset, + "inner": 1, + "isAjaxRequest": True, + }, + headers={ + "X-Requested-With": "XMLHttpRequest", + }, + ).json() + + soup = BeautifulSoup(response["content"], "html.parser") + urls = [ + tag["href"] + for tag in soup.find_all( + "a", + "inline-card-portalsubmission" + if conduct == Conduct.MOVIES + else "item-audiosubmission", + ) + ] + return self._fetch_metadata_batch(urls) + + def feed(self): + html = self._request( + "GET", + "https://www.newgrounds.com/social/feeds/show/favorite-artists-movies", + cookies=get_session(), + ).text + soup = BeautifulSoup(html, "html.parser") + urls = [tag["href"] for tag in soup.find_all("a", "item-portalsubmission")] + return self._fetch_metadata_batch(urls) + + def radio_status(self): + # https://stream.newgroundsradio.com/radio.mp3 + pass -- cgit v1.3