# Copyright © 2024 Jakob L. Kreuze # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ Video plugin for animations posted to Newgrounds. """ import concurrent.futures from datetime import datetime from dataclasses import dataclass, field from enum import Enum from dateutil import parser as dateutil_parser import json import os import sys from typing import List from urllib.parse import urlencode, parse_qsl from bs4 import BeautifulSoup import requests Suitability = Enum("Suitability", ["E", "T", "M", "A"]) Conduct = Enum( "Conduct", ["MOVIES", "AUDIO", "ART", "ALBUMS", "PLAYLISTS", "COLLECTIONS", "SERIES"], ) Grouping = Enum("Grouping", ["FEATURED", "LATEST", "POPULAR"]) SortBy = Enum( "SortBy", [ "RELEVANCE", "DATE_ASC", "DATE_DESC", "SCORE_ASC", "SCORE_DESC", "VIEWS_ASC", "VIEWS_DESC", ], ) Interval = Enum("Interval", ["TODAY", "YESTERDAY", "WEEK", "MONTH", "YEAR"]) AudioArchetype = Enum("AudioArchetype", ["MUSIC", "VOICE", "PODCASTS"]) def stringify_enum(enum_value): return enum_value.name.lower().replace("_", "-") @dataclass class VisualLink: title: str thumbnail: str author: str url: str @dataclass class Movie: title: str thumbnail: str content_url: str url: str views: int = 0 faves: int = 0 votes: int = 0 score: float = 0.0 upload_date: datetime = datetime.fromtimestamp(0) genre: str = "Unknown" tags: List[str] = field(default_factory=lambda: []) description: str = "" authors: List[str] = field(default_factory=lambda: []) suitability: Suitability = Suitability.E def __post_init__(self): pass def url(self): pass @dataclass class Audio: title: str thumbnail: str content_url: str url: str views: int = 0 faves: int = 0 votes: int = 0 score: float = 0.0 upload_date: datetime = datetime.fromtimestamp(0) genre: str = "Unknown" tags: list[str] = field(default_factory=lambda: []) description: str = "" authors: list[str] = field(default_factory=lambda: []) suitability: Suitability = Suitability.E def __post_init__(self): pass def url(self): pass MOVIE_CATEGORIES = { "All": 0, "Action": 45, "Comedy - Original": 60, "Comedy - Parody": 61, "Drama": 47, "Experimental": 49, "Informative": 48, "Music Video": 50, "Other": 51, "Spam": 55, } AUDIO_CATEGORIES = { "All": 0, "Easy Listening - Classical": 3, "Easy Listening - Jazz": 18, "Easy Listening - Solo Instrument": 51, "Electronic - Ambient": 5, "Electronic - Chipstep": 48, "Electronic - Dance": 6, "Electronic - Drum N Bass": 7, "Electronic - Dubstep": 41, "Electronic - House": 9, "Electronic - Industrial": 8, "Electronic - New Wave": 20, "Electronic - Synthwave": 57, "Electronic - Techno": 10, "Electronic - Trance": 11, "Electronic - Video Game": 12, "Hip Hop, Rap, R&B - Hip Hop - Modern": 17, "Hip Hop, Rap, R&B - Hip Hop - Olskool": 16, "Hip Hop, Rap, R&B - Nerdcore": 47, "Hip Hop, Rap, R&B - R&B": 21, "Metal, Rock - Brit Pop": 22, "Metal, Rock - Classic Rock": 23, "Metal, Rock - General Rock": 24, "Metal, Rock - Grunge": 25, "Metal, Rock - Heavy Metal": 15, "Metal, Rock - Indie": 26, "Metal, Rock - Pop": 27, "Metal, Rock - Punk": 28, "Other - Cinematic": 50, "Other - Experimental": 49, "Other - Funk": 13, "Other - Fusion": 52, "Other - Goth": 14, "Other - Miscellaneous": 39, "Other - Ska": 29, "Other - World": 19, "Southern Flavor - Bluegrass": 1, "Southern Flavor - Blues": 2, "Southern Flavor - Country": 4, } ART_CATEGORIES = { "All": 0, } SESSION_COOKIE_NAME = "vmkIdu5l8m" class Newgrounds(object): def __init__(self): pass def _request(self, method, url, **kwargs): kwargs = dict(kwargs) # TODO: Test if we need to open a new session? if hasattr(self, "session") and self.session is not None: if "cookies" in kwargs: kwargs["cookies"][SESSION_COOKIE_NAME] = self.session else: kwargs["cookies"] = {SESSION_COOKIE_NAME: self.session} r = requests.request(method, url, **kwargs) return r def login(self, username, password): self.session = None html = self._request("GET", "https://www.newgrounds.com/login").text soup = BeautifulSoup(html, "html.parser") form = soup.find("form", method="post") endpoint = form["action"] auth = form.find("input")["value"] r = self._request( "POST", "https://www.newgrounds.com" + endpoint, data={ "auth": auth, "username": username, "password": password, "code": "", "codehint": "------", }, cookies={ "passport-auth": auth, }, ) self.session = r.cookies[SESSION_COOKIE_NAME] def _test_logged_in(session): return ( requests.get( "https://www.newgrounds.com/account", cookies={SESSION_COOKIE_NAME: session}, ).status_code != 401 ) def series(self, official=True, sort_by=SortBy.DATE_DESC, offset=1): html = self._request( "GET", "https://www.newgrounds.com/series", params={ "filter": "all" if not official else None, "order": stringify_enum(sort_by), "page": offset, }, ).text soup = BeautifulSoup(html, "html.parser") visual_links = [ json.loads(li["data-visual-link"]) for li in soup.find_all("li", "visual-link-container") ] return self._fetch_visual_links(visual_links) def collections(self, official=True, sort_by=SortBy.DATE_DESC, offset=1): html = self._request( "GET", "https://www.newgrounds.com/collections", params={ "filter": "all" if not official else None, "order": stringify_enum(sort_by), "page": offset, }, ).text soup = BeautifulSoup(html, "html.parser") visual_links = [ json.loads(li["data-visual-link"]) for li in soup.find_all("li", "visual-link-container") ] return self._fetch_visual_links(visual_links) def playlist_entries(self, url): html = self._request("GET", url).text soup = BeautifulSoup(html, "html.parser") visual_links = [ json.loads(li["data-visual-link"]) for li in soup.find_all("li", "visual-link-container") ] urls = [card["url"] for card in self._fetch_visual_links(visual_links)] return self._fetch_metadata_batch(urls) def _fetch_metadata_batch(self, urls): result = [] with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: # Start the load operations and mark each future with its URL future_to_url = { executor.submit(self._fetch_metadata, url): url for url in urls } for future in concurrent.futures.as_completed(future_to_url): url = future_to_url[future] result.append(future.result()) return result def _fetch_visual_links(self, visual_links): r = self._request( "POST", "https://www.newgrounds.com/visual-links-fetch", params={ "X-Requested-With": "XMLHttpRequest", }, data={ "ids": json.dumps(visual_links).replace(" ", ""), "component_params[include_author]": "1", "include_all_suitabilities": "0", "isAjaxRequest": "1", }, ) result = r.json() entries = [] if isinstance(result["partials"], dict): for partial in result["partials"].values(): if isinstance(partial, dict): for partial in partial.values(): soup = BeautifulSoup(partial, "html.parser") title = soup.find("h4").string thumbnail = soup.find("img")["src"] author = soup.find("strong").string url = soup.find("a")["href"] entries.append(VisualLink(title, thumbnail, author, url)) return entries def _fetch_movie_content_url(self, movie_id): response = self._request( "GET", "https://www.newgrounds.com/portal/video/{}".format(movie_id), headers={"X-Requested-With": "XMLHttpRequest"}, ).json() preferences = ["1080p", "720p", "360p"] for preference in preferences: if preference in response["sources"]: source = next( filter( lambda x: x["type"] == "video/mp4", response["sources"][preference], ) ) if source is not None: return source["src"] def _fetch_movie_metadata(self, url): html = self._request("GET", url).text soup = BeautifulSoup(html, "html.parser") title = soup.find("div", id="embed_header").h2.string thumbnail = soup.find("meta", property="og:image") if thumbnail is not None: thumbnail = thumbnail["content"] movie_id = soup.find("meta", property="og:url")["content"].strip( "https://www.newgrounds.com/portal/view/" ) content_url = self._fetch_movie_content_url(movie_id) metadata = { "title": title, "thumbnail": thumbnail, "content_url": content_url, "url": url, } sidestats_popularity = soup.find_all("dl", "sidestats")[0].find_all("dd") if len(sidestats_popularity) == 4: metadata["views"] = int(sidestats_popularity[0].string.replace(",", "")) faves = sidestats_popularity[1].a if faves is not None: metadata["faves"] = int(faves.string.replace(",", "")) metadata["votes"] = int(sidestats_popularity[2].string.replace(",", "")) score = sidestats_popularity[3].span if score is not None: metadata["score"] = float(score.string) sidestats_date_and_genre = soup.find_all("dl", "sidestats")[1].find_all("dd") if len(sidestats_date_and_genre) == 3: metadata["upload_date"] = dateutil_parser.parse( sidestats_date_and_genre[0].string + " " + sidestats_date_and_genre[1].string ) genre = sidestats_date_and_genre[2].a if genre is not None: metadata["genre"] = genre.string tags = soup.find("dd", "tags") if tags is not None: metadata["tags"] = [tag.string for tag in tags.find_all("a")] author_comments = soup.find("div", id="author_comments") metadata["description"] = " ".join([s for s in author_comments.strings]).strip() authors = soup.find("ul", "authorlinks") metadata["authors"] = [ author.string for author in authors.find_all("a") if author.get("class") is None ] if len(soup.find_all("h2", "rated-a")) != 0: rating = "a" elif len(soup.find_all("h2", "rated-m")) != 0: rating = "m" elif len(soup.find_all("h2", "rated-t")) != 0: rating = "t" else: rating = "e" return Movie(**metadata) def _fetch_audio_metadata(self, url): html = self._request("GET", url).text soup = BeautifulSoup(html, "html.parser") title = soup.find("div", "pod-head").h2.string thumbnail = soup.find("meta", property="og:image") if thumbnail is not None: thumbnail = thumbnail["content"] content_url = soup.find("meta", property="og:audio")["content"] metadata = { "title": title, "thumbnail": thumbnail, "content_url": content_url, "url": url, } sidestats_popularity = soup.find_all("dl", "sidestats")[0].find_all("dd") if len(sidestats_popularity) == 4: metadata["views"] = int(sidestats_popularity[0].string.replace(",", "")) faves = sidestats_popularity[1].a if faves is not None: metadata["faves"] = int(faves.string.replace(",", "")) metadata["votes"] = int(sidestats_popularity[2].string.replace(",", "")) score = sidestats_popularity[3].span if score is not None: metadata["score"] = float(score.string) sidestats_date_and_genre = soup.find_all("dl", "sidestats")[1].find_all("dd") if len(sidestats_date_and_genre) >= 3: metadata["upload_date"] = dateutil_parser.parse( sidestats_date_and_genre[0].string + " " + sidestats_date_and_genre[1].string ) genre = sidestats_date_and_genre[2].a if genre is not None: metadata["genre"] = genre.string if len(sidestats_date_and_genre) >= 5: metadata["length"] = sidestats_date_and_genre[5].string.strip() tags = soup.find("dd", "tags") if tags is not None: metadata["tags"] = [tag.string for tag in tags.find_all("a")] # author_comments = soup.find("div", id="author_comments") # metadata["description"] = " ".join([s for s in author_comments.strings]).strip() authors = soup.find("div", "item-user") metadata["author"] = [ author.string for author in authors.find_all("a") if author.get("class") is None ][0] if len(soup.find_all("h2", "rated-a")) != 0: rating = "a" elif len(soup.find_all("h2", "rated-m")) != 0: rating = "m" elif len(soup.find_all("h2", "rated-t")) != 0: rating = "t" else: rating = "e" return Audio(**metadata) def _fetch_metadata(self, url): if "newgrounds.com/audio/" in url: return self._fetch_audio_metadata(url) elif "newgrounds.com/art/" in url: return self._fetch_art_metadata(url) else: return self._fetch_movie_metadata(url) def search(self, term, conduct=Conduct.MOVIES, sort_by=SortBy.RELEVANCE, offset=1): """ Return a list of URLs matching a given search term. """ response = self._request( "GET", "https://www.newgrounds.com/search/conduct/{}".format( stringify_enum(conduct) ), params={ "suitabilities": "etm", "sort": stringify_enum(sort_by), "terms": term, "page": offset, "inner": 1, }, headers={ "X-Requested-With": "XMLHttpRequest", }, ).json() soup = BeautifulSoup(response["content"], "html.parser") urls = [tag["href"] for tag in soup.find_all("a", "item-portalsubmission")] return self._fetch_metadata_batch(urls) def front_page( self, conduct=Conduct.MOVIES, grouping=Grouping.FEATURED, interval=Interval.TODAY, sort_by=SortBy.DATE_DESC, category="All", offset=0, ): # TODO: Support `after` parameter (e.g. `before=2024-06-04`) # TODO: Support `before` parameter (e.g. `before=2024-06-04`) URLS = { Grouping.FEATURED.value: "featured", Grouping.LATEST.value: "browse", Grouping.POPULAR.value: "popular", } assert conduct == Conduct.MOVIES or conduct == Conduct.AUDIO assert isinstance(category, str) assert isinstance(offset, int) if conduct == Conduct.MOVIES: assert category in MOVIE_CATEGORIES elif conduct == Conduct.AUDIO: assert category in AUDIO_CATEGORIES url = URLS[grouping] response = self._request( "GET", "https://www.newgrounds.com/movies/" + url if conduct == Conduct.MOVIES else "https://www.newgrounds.com/audio/" + url, params={ "interval": interval, "sort": stringify_enum(sort_by), "genre": category, "offset": offset, "inner": 1, "isAjaxRequest": True, }, headers={ "X-Requested-With": "XMLHttpRequest", }, ).json() soup = BeautifulSoup(response["content"], "html.parser") urls = [ tag["href"] for tag in soup.find_all( "a", "inline-card-portalsubmission" if conduct == Conduct.MOVIES else "item-audiosubmission", ) ] return self._fetch_metadata_batch(urls) def feed(self): html = self._request( "GET", "https://www.newgrounds.com/social/feeds/show/favorite-artists-movies", cookies=get_session(), ).text soup = BeautifulSoup(html, "html.parser") urls = [tag["href"] for tag in soup.find_all("a", "item-portalsubmission")] return self._fetch_metadata_batch(urls) def radio_status(self): # https://stream.newgroundsradio.com/radio.mp3 pass