From 278cd3d07dc14dda040cd9f16e327613c8d60e8c Mon Sep 17 00:00:00 2001 From: Jason Lobo Date: Sat, 3 Oct 2020 16:59:27 +0100 Subject: [PATCH] Create sources.py --- sources.py | 346 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100644 sources.py diff --git a/sources.py b/sources.py new file mode 100644 index 0000000..c1acc5d --- /dev/null +++ b/sources.py @@ -0,0 +1,346 @@ +import json +import subprocess + +def urls_yt_add(): + urls_yt_add = [ + + + # Channels + + # The Forest Revealed + 'https://www.youtube.com/c/TheForestRevealed/videos', + # Chomsky's Philosophy + 'https://www.youtube.com/user/chomskysphilosophy/videos', + # Ricardo Ruiz + 'https://www.youtube.com/channel/UCXZXj5KotSzQBQDKdK-o49Q', + # Chomskyan + 'https://www.youtube.com/user/Chomskyan/videos', + # Digital Chomsky + 'https://www.youtube.com/channel/UCvec0zvpmnxdfKh5BZMavRQ', + # letNOAMspeak + 'https://www.youtube.com/user/letNOAMspeak/videos', + # noamychomskyy + 'https://www.youtube.com/user/noamychomskyy/videos', + + + # Playlists + + # Chomsky Archive + 'https://www.youtube.com/playlist?list=PLJAP0acmX6BmeJ7Yktv9kmzlJ-dLHdD1h', + # Cognitive Revolution + 'https://www.youtube.com/playlist?list=PL6nOdowQ4jwmJAAtK5ixnl5v4EoBqzMhS', + # Misc + 'https://www.youtube.com/playlist?list=PL13E62E4DC420B97A', + # Nationalism + 'https://www.youtube.com/playlist?list=PL2DT1k6e9kKX_cC8WEx369U9hSAiAUNGh', + # Democracy Now! + 'https://www.youtube.com/playlist?list=PL8E296CDF21DE9D35', + # Linguistics + 'https://www.youtube.com/playlist?list=PLHU1ryHGd_xaSQfxOmaqf-oueT2WO5aQE', + # infiniteinfiniteinfi + 'https://www.youtube.com/playlist?list=PL8B20BDCE43D7A2BA', + # Misc + 'https://www.youtube.com/playlist?list=PLi6hnQx6eEsARtpAp7BPOruiNnE2XCGIY', + # Misc + 'https://www.youtube.com/playlist?list=PL528C06FF92A3DFF0', + # Misc + 'https://www.youtube.com/playlist?list=PLi6hnQx6eEsCvp_SR50-uK_QD2YU0xylW', + # Misc + 'https://www.youtube.com/playlist?list=PLjRJEmwDWKwrC4qLrfZP5k6aY7ui0p0eJ', + # Chomsky Sessions + 'https://www.youtube.com/playlist?list=PLn5kIRhssBsn6CqX4w7fhWozb5oUwu1ap', + # Misc + 'https://www.youtube.com/playlist?list=PLbfLNPEPw2rKwaQAn_h9DB5pHZmofFhh-', + # Boston College + 'https://www.youtube.com/playlist?list=PLDAC3BFD45C357BF1', + + + # Videos + + 'https://www.youtube.com/watch?v=hrhFI1rsDDE', + 'https://www.youtube.com/watch?v=oJSQ7_I7zmw', + 'https://www.youtube.com/watch?v=r514RhgISv0', + 'https://www.youtube.com/watch?v=GPHew_smDjY', + 'https://www.youtube.com/watch?v=Lv4QSpY3cbY', + 'https://www.youtube.com/watch?v=fHvhWZJMPwc', + 'https://www.youtube.com/watch?v=hrhFI1rsDDE', + 'https://www.youtube.com/watch?v=6XZ29-kGCmg', + 'https://www.youtube.com/watch?v=kfvtD4ptZ-k', + 'https://www.youtube.com/watch?v=qDjXsdslrYc', + 'https://www.youtube.com/watch?v=XeWWz4y1coU', + 'https://www.youtube.com/watch?v=apfgfG-Gmp4', + 'https://www.youtube.com/watch?v=Vmh10WtC7ws', + 'https://www.youtube.com/watch?v=_AMzzKonrTQ', + 'https://www.youtube.com/watch?v=owp9ooAbeXc', + 'https://www.youtube.com/watch?v=EA7Dj-dmzEE', + 'https://www.youtube.com/watch?v=DJKf9wKSbZ4', + 'https://www.youtube.com/watch?v=cMscNuSUy0I', + 'https://www.youtube.com/watch?v=8ED1Xt3IoRc', + 'https://www.youtube.com/watch?v=yuVqfKYbGvE', + 'https://www.youtube.com/watch?v=SL2IpMNTz1A', + 'https://www.youtube.com/watch?v=mpIF7xjGyRs', + 'https://www.youtube.com/watch?v=xebTB_Lh8I8', + 'https://www.youtube.com/watch?v=xgYbM2Q9cso', + 'https://www.youtube.com/watch?v=WGdljrhdSKA', + 'https://www.youtube.com/watch?v=Ud96nzm5Dvk', + 'https://www.youtube.com/watch?v=9B_eLoVq4yI', + 'https://www.youtube.com/watch?v=KZthDGH6WCU', + 'https://www.youtube.com/watch?v=xOVuhgx0A0E', + 'https://www.youtube.com/watch?v=XXM4Py1UA3g', + 'https://www.youtube.com/watch?v=widzhMeoAy0', + 'https://www.youtube.com/watch?v=ea1p1eevcfo', + 'https://www.youtube.com/watch?v=t-N3In2rLI4', + 'https://www.youtube.com/watch?v=8wb6LFOYyQQ', + 'https://www.youtube.com/watch?v=kP9pUqqeUD0', + 'https://www.youtube.com/watch?v=OEdfJP9S4NE', + 'https://www.youtube.com/watch?v=rQLq9iLQseA', + 'https://www.youtube.com/watch?v=A5MQw3N53MY', + 'https://www.youtube.com/watch?v=2oqQgQYBWvI', + 'https://www.youtube.com/watch?v=8TNOq7zgBZ8', + 'https://www.youtube.com/watch?v=VJFbxe21y1M', + 'https://www.youtube.com/watch?v=unWjIwBarIY', + 'https://www.youtube.com/watch?v=kwN5BV8vZUg', + 'https://www.youtube.com/watch?v=b2VyTIMhUqw', + 'https://www.youtube.com/watch?v=d-6EXZ7yM2E', + 'https://www.youtube.com/watch?v=39902cn5lX8', + 'https://www.youtube.com/watch?v=lciwaaFQqCk', + 'https://www.youtube.com/watch?v=gP6T_-bqt40', + 'https://www.youtube.com/watch?v=MHhgCFJWgM0', + 'https://www.youtube.com/watch?v=w-wSasUe1hc', + 'https://www.youtube.com/watch?v=BW0l2ESybiY', + 'https://www.youtube.com/watch?v=1_dBnu4MUIQ', + 'https://www.youtube.com/watch?v=twUinwhHL-k', + 'https://www.youtube.com/watch?v=zi6ae6kZNqE', + 'https://www.youtube.com/watch?v=Vq6IgBMg3lI', + 'https://www.youtube.com/watch?v=k1kqgTKf0kw', + 'https://www.youtube.com/watch?v=hX3ZbF0i2aE', + 'https://www.youtube.com/watch?v=HO0OpQN9d1s', + 'https://www.youtube.com/watch?v=VLwjqrfRAYM', + 'https://www.youtube.com/watch?v=Rtt3d0mtJe0', + 'https://www.youtube.com/watch?v=SO3tE7clWhI', + 'https://www.youtube.com/watch?v=ZXfXy771K8w', + 'https://www.youtube.com/watch?v=VUYYZdE5Dmk', + 'https://www.youtube.com/watch?v=j7B6vVv57aw', + 'https://www.youtube.com/watch?v=Rye4tVkm4iA', + 'https://www.youtube.com/watch?v=C9tw09IzQdM', + 'https://www.youtube.com/watch?v=4MM55B0zMB4', + 'https://www.youtube.com/watch?v=O2dyYT3SdP8', + 'https://www.youtube.com/watch?v=heGfKTRbWfo', + 'https://www.youtube.com/watch?v=LQOeDg0qFf4', + 'https://www.youtube.com/watch?v=mFeCq-KZ9nM', + 'https://www.youtube.com/watch?v=n_LD6alcsn0', + 'https://www.youtube.com/watch?v=RiTl0moYYeQ', + 'https://www.youtube.com/watch?v=jgPyTXw65l0', + 'https://www.youtube.com/watch?v=Jr8f6f0cnuc', + 'https://www.youtube.com/watch?v=6UYbGA6yB7Q', + 'https://www.youtube.com/watch?v=sb_TM5uUoSM', + 'https://www.youtube.com/watch?v=SZBiZHpK398', + 'https://www.youtube.com/watch?v=IqfWUe0UbZA', + 'https://www.youtube.com/watch?v=vaF-W2Wg0Pg', + 'https://www.youtube.com/watch?v=MER4y8AMjrc', + 'https://www.youtube.com/watch?v=aR56qEj1u58', + 'https://www.youtube.com/watch?v=0eqiQNPbe8I', + 'https://www.youtube.com/watch?v=DYZXrzEvoIM', + 'https://www.youtube.com/watch?v=9bIgZ_ILn6I', + 'https://www.youtube.com/watch?v=TM8KRHnTI_M', + 'https://www.youtube.com/watch?v=HfG5IyMc89w', + 'https://www.youtube.com/watch?v=YbYMVyuuRvY', + 'https://www.youtube.com/watch?v=V0pthiN2Clc', + 'https://www.youtube.com/watch?v=LTPuJ541N0o', + 'https://www.youtube.com/watch?v=O4L8f2GI1NM', + 'https://www.youtube.com/watch?v=UH0v9pjo-I0', + 'https://www.youtube.com/watch?v=efRsPcDuHmE', + 'https://www.youtube.com/watch?v=g-HfLkiZsYg', + 'https://www.youtube.com/watch?v=hCdGvWuF1d8', + 'https://www.youtube.com/watch?v=Iaz6JIxDh6Y', + 'https://www.youtube.com/watch?v=mn_MaruQIfo', + 'https://www.youtube.com/watch?v=0bNIwlODieI', + 'https://www.youtube.com/watch?v=jnZUHQHGTYM', + 'https://www.youtube.com/watch?v=uiCeqySVhCE', + 'https://www.youtube.com/watch?v=dd96_vUNsYQ', + 'https://www.youtube.com/watch?v=YlUfufBj5vU', + 'https://www.youtube.com/watch?v=X6mAf_uPy-o', + 'https://www.youtube.com/watch?v=8LqzVjev4vg', + 'https://www.youtube.com/watch?v=zRvqkUoiKJo', + 'https://www.youtube.com/watch?v=URZihNBsnjA', + 'https://www.youtube.com/watch?v=dg5ciDyauI4', + 'https://www.youtube.com/watch?v=4plIX68mdE8', + 'https://www.youtube.com/watch?v=ba_cJYpONvw', + 'https://www.youtube.com/watch?v=SVk3Nh_seq0', + 'https://www.youtube.com/watch?v=8ZWS0A5m8f8', + 'https://www.youtube.com/watch?v=TZeCBVvH3bU', + 'https://www.youtube.com/watch?v=WBSi93dzxPA', + 'https://www.youtube.com/watch?v=OWhbSKlN3fE', + 'https://www.youtube.com/watch?v=YvcCyRGOIYw', + 'https://www.youtube.com/watch?v=3lcDT_-3v2k', + 'https://www.youtube.com/watch?v=TYAjo3DCY6M', + 'https://www.youtube.com/watch?v=ZiPXFeL3KQU', + 'https://www.youtube.com/watch?v=cr69xk1xiOU', + + 'https://www.youtube.com/watch?v=vq9irdLcZmU', + 'https://www.youtube.com/watch?v=PHGuhEvdg6M', + 'https://www.youtube.com/watch?v=jnc1Ay6X1bg', + 'https://www.youtube.com/watch?v=hweZuG-UBmM', + 'https://www.youtube.com/watch?v=EXBNGlUVgFs', + 'https://www.youtube.com/watch?v=YhSUknaEnec', + 'https://www.youtube.com/watch?v=W7QBt61ChR4', + 'https://www.youtube.com/watch?v=kVNq8knHGew', + 'https://www.youtube.com/watch?v=F5pJR45HNEM', + 'https://www.youtube.com/watch?v=D5in5EdjhD0', + 'https://www.youtube.com/watch?v=rV2KFnsfYPk', + 'https://www.youtube.com/watch?v=DFaNajmJ2oc', + 'https://www.youtube.com/watch?v=RMxKc-uFEnU', + 'https://www.youtube.com/watch?v=1pH8gCEZ4Ds', + 'https://www.youtube.com/watch?v=oUUkwmTmYqM', + 'https://www.youtube.com/watch?v=CHS1NraVsAc', + 'https://www.youtube.com/watch?v=U7EyfO0TRm4', + 'https://www.youtube.com/watch?v=0XnihgXhh3c', + 'https://www.youtube.com/watch?v=9DvmLMUfGss', + 'https://www.youtube.com/watch?v=XNSxj0TVeJs', + 'https://www.youtube.com/watch?v=-U56oPQado8', + 'https://www.youtube.com/watch?v=2qCaW1_4LBQ', + 'https://www.youtube.com/watch?v=uEGVwVaauDM', + 'https://www.youtube.com/watch?v=2G6kf7XM9Nk', + 'https://www.youtube.com/watch?v=2HvGy2gY0eM', + 'https://www.youtube.com/watch?v=Grn2EZxcUeA', + 'https://www.youtube.com/watch?v=-tU918oK85o', + 'https://www.youtube.com/watch?v=Y3PwG4UoJ0Y', + 'https://www.youtube.com/watch?v=h00PgJSL8wY', + 'https://www.youtube.com/watch?v=8NTWv4qr3iA', + 'https://www.youtube.com/watch?v=StGGngQIwY0', + 'https://www.youtube.com/watch?v=L7ZK8J69CW0', + 'https://www.youtube.com/watch?v=PHEKSSpOZGc', + 'https://www.youtube.com/watch?v=2yVQOCzw34U', + 'https://www.youtube.com/watch?v=5adE7kHqTLU', + 'https://www.youtube.com/watch?v=guPlS0___ww', + 'https://www.youtube.com/watch?v=Ah5RsGc5Tnk', + 'https://www.youtube.com/watch?v=86RY40TvRgE', + 'https://www.youtube.com/watch?v=-NGQ9INMnq0', + 'https://www.youtube.com/watch?v=szIGZVrSAyc', + 'https://www.youtube.com/watch?v=CvZRsdHgxgA', + 'https://www.youtube.com/watch?v=mk8pxyAWTBk', + 'https://www.youtube.com/watch?v=KYlyb1Bx9Ic', + 'https://www.youtube.com/watch?v=8-jh2R-_eQY', + 'https://www.youtube.com/watch?v=MXh1_ubCQAI', + 'https://www.youtube.com/watch?v=Zh7ps1KWl_M', + 'https://www.youtube.com/watch?v=uXPvPxOFba0', + 'https://www.youtube.com/watch?v=-3kAld2YwgU', + 'https://www.youtube.com/watch?v=nvo9O4A18d0', + 'https://www.youtube.com/watch?v=5OA-KgruEEg', + 'https://www.youtube.com/watch?v=fy69-u4rcHU', + 'https://www.youtube.com/watch?v=4vGc0xdZsf4', + 'https://www.youtube.com/watch?v=K_Gym829MIM', + 'https://www.youtube.com/watch?v=GL1GcRrbPm0', + 'https://www.youtube.com/watch?v=e8TihISaf7E', + 'https://www.youtube.com/watch?v=9JScy7ulDpE', + 'https://www.youtube.com/watch?v=TAP0xk-c4mk', + 'https://www.youtube.com/watch?v=u0t34c3W8d0', + 'https://www.youtube.com/watch?v=e1JWr03P9W8', + 'https://www.youtube.com/watch?v=H-fRSrfr2L0', + 'https://www.youtube.com/watch?v=tKFgMHFXaYs', + 'https://www.youtube.com/watch?v=6gD-W-X6hdc', + 'https://www.youtube.com/watch?v=WpJjuotD534', + 'https://www.youtube.com/watch?v=VBI9mC77igo', + 'https://www.youtube.com/watch?v=WSN_8YGX6W8', + 'https://www.youtube.com/watch?v=QKq38COoTG8', + 'https://www.youtube.com/watch?v=bgqwfCyZpao', + 'https://www.youtube.com/watch?v=7TLZN92-dZo', + 'https://www.youtube.com/watch?v=VoRmqbPFasE', + 'https://www.youtube.com/watch?v=HDXmWCzw5_U' + + + ] + + return urls_yt_add + +def urls_yt_remove(): + urls_yt_remove = [ + # The Forest Revealed + 'https://www.youtube.com/watch?v=qAAX4olzVuQ', + 'https://www.youtube.com/watch?v=KafxQqt-Ntw', + 'https://www.youtube.com/watch?v=2s70bDIMs8c', + 'https://www.youtube.com/watch?v=p4y8fTl00tI', + 'https://www.youtube.com/watch?v=MKxXyt-YTss', + 'https://www.youtube.com/watch?v=_mpZplkvm-s', + 'https://www.youtube.com/watch?v=cOVl0ub0Mls', + 'https://www.youtube.com/watch?v=kejMQ0tcdhs', + 'https://www.youtube.com/watch?v=8x6uvAH0Yr8', + 'https://www.youtube.com/watch?v=0M1G5DLnmfA', + 'https://www.youtube.com/watch?v=0kpaVxnbWh4', + 'https://www.youtube.com/watch?v=96Zt9l53xWk', + 'https://www.youtube.com/watch?v=R6dZ_obn1NU', + 'https://www.youtube.com/watch?v=38uAAYHGPYo', + 'https://www.youtube.com/watch?v=yJzCNfdTf60', + 'https://www.youtube.com/watch?v=7eNEfeIytMs', + 'https://www.youtube.com/watch?v=OGQZtnyi4hc', + 'https://www.youtube.com/watch?v=E6vLshKTOxQ', + 'https://www.youtube.com/watch?v=-bLb7VZnqko', + + # Ricardo Ruiz + 'https://www.youtube.com/watch?v=AUniGUu5hGE', + 'https://www.youtube.com/watch?v=CooK_RWMreI', + 'https://www.youtube.com/watch?v=05XGb2Aw5lA', + 'https://www.youtube.com/watch?v=SDZYRvwI7pY', + 'https://www.youtube.com/watch?v=VZYNRKKuaXY', + + # Chomskyan + 'https://www.youtube.com/watch?v=HWgv0Pr1i9o', + 'https://www.youtube.com/watch?v=7-iGi4H2Vqs', + 'https://www.youtube.com/watch?v=UTnd8jG9Lfo', + 'https://www.youtube.com/watch?v=eYr2wcsNGVc', + 'https://www.youtube.com/watch?v=a27mnDbHCow', + 'https://www.youtube.com/watch?v=4tfkiUtB3_0', + 'https://www.youtube.com/watch?v=QNLHDsZZ50Q', + 'https://www.youtube.com/watch?v=zIERifyW_aI', + + # Linguistics + 'https://www.youtube.com/watch?v=ycC5oZqNXsk', + 'https://www.youtube.com/watch?v=yRSxR9Us9QM', + 'https://www.youtube.com/watch?v=msOnzHPeXzE', + 'https://www.youtube.com/watch?v=24PcF7LDQ7M', + + # ...XCGIY + 'https://www.youtube.com/watch?v=0lupJCD8DEw', + + # ...0xylW + 'https://www.youtube.com/watch?v=O4GLzz-9lRY', + 'https://www.youtube.com/watch?v=6-d23ma3qz8', + 'https://www.youtube.com/watch?v=XUBYI97cUgU', + 'https://www.youtube.com/watch?v=ww__v9i-PU4', + 'https://www.youtube.com/watch?v=lWyuKzH9NSQ', + 'https://www.youtube.com/watch?v=IyRoVOSVqZ8', + + # Unknown + 'https://www.youtube.com/watch?v=wlrOyum4US0', + 'https://www.youtube.com/watch?v=34LGPIXvU5M' + + ] + + return urls_yt_remove + + +# Remove duplicates from across YT channels/playlists/videos +# by using youtube-dl to create lists of video ids +def urls_yt(): + + yt_add_list = [] + + for url in urls_yt_add(): + output = subprocess.check_output('powershell.exe ' + './youtube-dl -j --flat-playlist ' + url, stderr=subprocess.STDOUT, shell=True) + for jsonObj in output.decode('UTF-8').splitlines(): + yt_add_dict = json.loads(jsonObj) + yt_add_list.append(yt_add_dict["id"]) + + yt_add_set = set(yt_add_list) + + yt_remove_list = [] + + for url in urls_yt_remove(): + output = subprocess.check_output('powershell.exe ' + './youtube-dl -j --flat-playlist ' + url, stderr=subprocess.STDOUT, shell=True) + for jsonObj in output.decode('UTF-8').splitlines(): + yt_remove_dict = json.loads(jsonObj) + yt_remove_list.append(yt_remove_dict["id"]) + + yt_remove_set = set(yt_remove_list) + + return yt_add_set.difference(yt_remove_set)