1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
import pytest
from szurubooru import errors
from szurubooru.func import net
from szurubooru.func.util import get_sha1
@pytest.fixture(autouse=True)
def inject_config(tmpdir, config_injector):
config_injector(
{
"user_agent": None,
"max_dl_filesize": 1.0e6,
"data_dir": str(tmpdir.mkdir("data")),
}
)
def test_download():
url = "http://info.cern.ch/hypertext/WWW/TheProject.html"
expected_content = (
b'<HEADER>\n<TITLE>The World Wide Web project</TITLE>\n<NEXTID N="'
+ b'55">\n</HEADER>\n<BODY>\n<H1>World Wide Web</H1>The WorldWideWeb'
+ b' (W3) is a wide-area<A\nNAME=0 HREF="WhatIs.html">\nhypermedia</'
+ b"A> information retrieval\ninitiative aiming to give universal\na"
+ b"ccess to a large universe of documents.<P>\nEverything there is "
+ b"online about\nW3 is linked directly or indirectly\nto this docum"
+ b'ent, including an <A\nNAME=24 HREF="Summary.html">executive\nsum'
+ b'mary</A> of the project, <A\nNAME=29 HREF="Administration/Mailin'
+ b'g/Overview.html">Mailing lists</A>\n, <A\nNAME=30 HREF="Policy.h'
+ b'tml">Policy</A> , November\'s <A\nNAME=34 HREF="News/9211.html"'
+ b'>W3 news</A> ,\n<A\nNAME=41 HREF="FAQ/List.html">Frequently Ask'
+ b'ed Questions</A> .\n<DL>\n<DT><A\nNAME=44 HREF="../DataSources/T'
+ b"op.html\">What's out there?</A>\n<DD> Pointers to the\nworld's "
+ b'online information,<A\nNAME=45 HREF="../DataSources/bySubject/Ov'
+ b'erview.html"> subjects</A>\n, <A\nNAME=z54 HREF="../DataSources/'
+ b'WWW/Servers.html">W3 servers</A>, etc.\n<DT><A\nNAME=46 HREF="He'
+ b'lp.html">Help</A>\n<DD> on the browser you are using\n<DT><A\nNA'
+ b'ME=13 HREF="Status.html">Software Products</A>\n<DD> A list of W'
+ b"3 project\ncomponents and their current state.\n(e.g. <A\nNAME=2"
+ b'7 HREF="LineMode/Browser.html">Line Mode</A> ,X11 <A\nNAME=35 HR'
+ b'EF="Status.html#35">Viola</A> , <A\nNAME=26 HREF="NeXT/WorldWid'
+ b'eWeb.html">NeXTStep</A>\n, <A\nNAME=25 HREF="Daemon/Overview.htm'
+ b'l">Servers</A> , <A\nNAME=51 HREF="Tools/Overview.html">Tools</A'
+ b'> ,<A\nNAME=53 HREF="MailRobot/Overview.html"> Mail robot</A> ,<'
+ b'A\nNAME=52 HREF="Status.html#57">\nLibrary</A> )\n<DT><A\nNAME=4'
+ b'7 HREF="Technical.html">Technical</A>\n<DD> Details of protocols'
+ b', formats,\nprogram internals etc\n<DT><A\nNAME=40 HREF="Bibliog'
+ b'raphy.html">Bibliography</A>\n<DD> Paper documentation\non W3 a'
+ b'nd references.\n<DT><A\nNAME=14 HREF="People.html">People</A>\n<'
+ b"DD> A list of some people involved\nin the project.\n<DT><A\nNAM"
+ b'E=15 HREF="History.html">History</A>\n<DD> A summary of the hist'
+ b'ory\nof the project.\n<DT><A\nNAME=37 HREF="Helping.html">How ca'
+ b"n I help</A> ?\n<DD> If you would like\nto support the web..\n<D"
+ b'T><A\nNAME=48 HREF="../README.html">Getting code</A>\n<DD> Getti'
+ b'ng the code by<A\nNAME=49 HREF="LineMode/Defaults/Distribution.h'
+ b'tml">\nanonymous FTP</A> , etc.</A>\n</DL>\n</BODY>\n'
)
actual_content = net.download(url)
assert actual_content == expected_content
@pytest.mark.parametrize(
"url",
[
"https://samples.ffmpeg.org/MPEG-4/video.mp4",
"https://www.youtube.com/watch?v=dQw4w9WgXcQ",
],
)
def test_too_large_download(url):
with pytest.raises(net.DownloadTooLargeError):
net.download(url, use_video_downloader=True)
@pytest.mark.parametrize(
"url,expected_sha1",
[
(
"https://gfycat.com/immaterialchillyiberianmole",
"0125976d2439e651b6863438db30de58f79f7754",
),
(
"https://upload.wikimedia.org/wikipedia/commons/a/ad/Utah_teapot.png", # noqa: E501
"cfadcbdeda1204dc1363ee5c1969191f26be2e41",
),
(
"https://i.imgur.com/GPgh0AN.jpg",
"26861a4663fedae48e5beed3eec5156ded20640f",
),
],
)
def test_content_download(url, expected_sha1):
actual_content = net.download(url, use_video_downloader=True)
assert get_sha1(actual_content) == expected_sha1
def test_bad_content_downlaod():
url = "http://info.cern.ch/hypertext/WWW/TheProject.html"
with pytest.raises(errors.ThirdPartyError):
net.download(url, use_video_downloader=True)
def test_no_webhooks(config_injector):
config_injector({"webhooks": []})
res = net.post_to_webhooks(None)
assert len(res) == 0
@pytest.mark.parametrize(
"webhook,status_code",
[
("https://postman-echo.com/post", 200),
("http://localhost/", 400),
("https://postman-echo.com/get", 400),
],
)
def test_single_webhook(config_injector, webhook, status_code):
ret = net._post_to_webhook(webhook, {"test_arg": "test_value"})
assert ret == status_code
def test_multiple_webhooks(config_injector):
config_injector(
{
"webhooks": [
"https://postman-echo.com/post",
"https://postman-echo.com/get",
]
}
)
threads = net.post_to_webhooks({"test_arg": "test_value"})
assert len(threads) == 2
def test_malformed_webhooks(config_injector):
with pytest.raises(ValueError):
net._post_to_webhook("malformed_url", {"test_arg": "test_value"})
|
