prehistoric-systems/the-chomsky-index
1
0
Fork 0
mirror of https://github.com/jasons-gh/the-chomsky-index.git synced 2026-03-11 08:54:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
the-chomsky-index/sources.py
Jason Lobo 278cd3d07d Create sources.py
2020-10-03 16:59:27 +01:00

346 lines
16 KiB
Python
Raw Blame History

import json
import subprocess
def urls_yt_add():
urls_yt_add = [
# Channels
# The Forest Revealed
'https://www.youtube.com/c/TheForestRevealed/videos',
# Chomsky's Philosophy
'https://www.youtube.com/user/chomskysphilosophy/videos',
# Ricardo Ruiz
'https://www.youtube.com/channel/UCXZXj5KotSzQBQDKdK-o49Q',
# Chomskyan
'https://www.youtube.com/user/Chomskyan/videos',
# Digital Chomsky
'https://www.youtube.com/channel/UCvec0zvpmnxdfKh5BZMavRQ',
# letNOAMspeak
'https://www.youtube.com/user/letNOAMspeak/videos',
# noamychomskyy
'https://www.youtube.com/user/noamychomskyy/videos',
# Playlists
# Chomsky Archive
'https://www.youtube.com/playlist?list=PLJAP0acmX6BmeJ7Yktv9kmzlJ-dLHdD1h',
# Cognitive Revolution
'https://www.youtube.com/playlist?list=PL6nOdowQ4jwmJAAtK5ixnl5v4EoBqzMhS',
# Misc
'https://www.youtube.com/playlist?list=PL13E62E4DC420B97A',
# Nationalism
'https://www.youtube.com/playlist?list=PL2DT1k6e9kKX_cC8WEx369U9hSAiAUNGh',
# Democracy Now!
'https://www.youtube.com/playlist?list=PL8E296CDF21DE9D35',
# Linguistics
'https://www.youtube.com/playlist?list=PLHU1ryHGd_xaSQfxOmaqf-oueT2WO5aQE',
# infiniteinfiniteinfi
'https://www.youtube.com/playlist?list=PL8B20BDCE43D7A2BA',
# Misc
'https://www.youtube.com/playlist?list=PLi6hnQx6eEsARtpAp7BPOruiNnE2XCGIY',
# Misc
'https://www.youtube.com/playlist?list=PL528C06FF92A3DFF0',
# Misc
'https://www.youtube.com/playlist?list=PLi6hnQx6eEsCvp_SR50-uK_QD2YU0xylW',
# Misc
'https://www.youtube.com/playlist?list=PLjRJEmwDWKwrC4qLrfZP5k6aY7ui0p0eJ',
# Chomsky Sessions
'https://www.youtube.com/playlist?list=PLn5kIRhssBsn6CqX4w7fhWozb5oUwu1ap',
# Misc
'https://www.youtube.com/playlist?list=PLbfLNPEPw2rKwaQAn_h9DB5pHZmofFhh-',
# Boston College
'https://www.youtube.com/playlist?list=PLDAC3BFD45C357BF1',
# Videos
'https://www.youtube.com/watch?v=hrhFI1rsDDE',
'https://www.youtube.com/watch?v=oJSQ7_I7zmw',
'https://www.youtube.com/watch?v=r514RhgISv0',
'https://www.youtube.com/watch?v=GPHew_smDjY',
'https://www.youtube.com/watch?v=Lv4QSpY3cbY',
'https://www.youtube.com/watch?v=fHvhWZJMPwc',
'https://www.youtube.com/watch?v=hrhFI1rsDDE',
'https://www.youtube.com/watch?v=6XZ29-kGCmg',
'https://www.youtube.com/watch?v=kfvtD4ptZ-k',
'https://www.youtube.com/watch?v=qDjXsdslrYc',
'https://www.youtube.com/watch?v=XeWWz4y1coU',
'https://www.youtube.com/watch?v=apfgfG-Gmp4',
'https://www.youtube.com/watch?v=Vmh10WtC7ws',
'https://www.youtube.com/watch?v=_AMzzKonrTQ',
'https://www.youtube.com/watch?v=owp9ooAbeXc',
'https://www.youtube.com/watch?v=EA7Dj-dmzEE',
'https://www.youtube.com/watch?v=DJKf9wKSbZ4',
'https://www.youtube.com/watch?v=cMscNuSUy0I',
'https://www.youtube.com/watch?v=8ED1Xt3IoRc',
'https://www.youtube.com/watch?v=yuVqfKYbGvE',
'https://www.youtube.com/watch?v=SL2IpMNTz1A',
'https://www.youtube.com/watch?v=mpIF7xjGyRs',
'https://www.youtube.com/watch?v=xebTB_Lh8I8',
'https://www.youtube.com/watch?v=xgYbM2Q9cso',
'https://www.youtube.com/watch?v=WGdljrhdSKA',
'https://www.youtube.com/watch?v=Ud96nzm5Dvk',
'https://www.youtube.com/watch?v=9B_eLoVq4yI',
'https://www.youtube.com/watch?v=KZthDGH6WCU',
'https://www.youtube.com/watch?v=xOVuhgx0A0E',
'https://www.youtube.com/watch?v=XXM4Py1UA3g',
'https://www.youtube.com/watch?v=widzhMeoAy0',
'https://www.youtube.com/watch?v=ea1p1eevcfo',
'https://www.youtube.com/watch?v=t-N3In2rLI4',
'https://www.youtube.com/watch?v=8wb6LFOYyQQ',
'https://www.youtube.com/watch?v=kP9pUqqeUD0',
'https://www.youtube.com/watch?v=OEdfJP9S4NE',
'https://www.youtube.com/watch?v=rQLq9iLQseA',
'https://www.youtube.com/watch?v=A5MQw3N53MY',
'https://www.youtube.com/watch?v=2oqQgQYBWvI',
'https://www.youtube.com/watch?v=8TNOq7zgBZ8',
'https://www.youtube.com/watch?v=VJFbxe21y1M',
'https://www.youtube.com/watch?v=unWjIwBarIY',
'https://www.youtube.com/watch?v=kwN5BV8vZUg',
'https://www.youtube.com/watch?v=b2VyTIMhUqw',
'https://www.youtube.com/watch?v=d-6EXZ7yM2E',
'https://www.youtube.com/watch?v=39902cn5lX8',
'https://www.youtube.com/watch?v=lciwaaFQqCk',
'https://www.youtube.com/watch?v=gP6T_-bqt40',
'https://www.youtube.com/watch?v=MHhgCFJWgM0',
'https://www.youtube.com/watch?v=w-wSasUe1hc',
'https://www.youtube.com/watch?v=BW0l2ESybiY',
'https://www.youtube.com/watch?v=1_dBnu4MUIQ',
'https://www.youtube.com/watch?v=twUinwhHL-k',
'https://www.youtube.com/watch?v=zi6ae6kZNqE',
'https://www.youtube.com/watch?v=Vq6IgBMg3lI',
'https://www.youtube.com/watch?v=k1kqgTKf0kw',
'https://www.youtube.com/watch?v=hX3ZbF0i2aE',
'https://www.youtube.com/watch?v=HO0OpQN9d1s',
'https://www.youtube.com/watch?v=VLwjqrfRAYM',
'https://www.youtube.com/watch?v=Rtt3d0mtJe0',
'https://www.youtube.com/watch?v=SO3tE7clWhI',
'https://www.youtube.com/watch?v=ZXfXy771K8w',
'https://www.youtube.com/watch?v=VUYYZdE5Dmk',
'https://www.youtube.com/watch?v=j7B6vVv57aw',
'https://www.youtube.com/watch?v=Rye4tVkm4iA',
'https://www.youtube.com/watch?v=C9tw09IzQdM',
'https://www.youtube.com/watch?v=4MM55B0zMB4',
'https://www.youtube.com/watch?v=O2dyYT3SdP8',
'https://www.youtube.com/watch?v=heGfKTRbWfo',
'https://www.youtube.com/watch?v=LQOeDg0qFf4',
'https://www.youtube.com/watch?v=mFeCq-KZ9nM',
'https://www.youtube.com/watch?v=n_LD6alcsn0',
'https://www.youtube.com/watch?v=RiTl0moYYeQ',
'https://www.youtube.com/watch?v=jgPyTXw65l0',
'https://www.youtube.com/watch?v=Jr8f6f0cnuc',
'https://www.youtube.com/watch?v=6UYbGA6yB7Q',
'https://www.youtube.com/watch?v=sb_TM5uUoSM',
'https://www.youtube.com/watch?v=SZBiZHpK398',
'https://www.youtube.com/watch?v=IqfWUe0UbZA',
'https://www.youtube.com/watch?v=vaF-W2Wg0Pg',
'https://www.youtube.com/watch?v=MER4y8AMjrc',
'https://www.youtube.com/watch?v=aR56qEj1u58',
'https://www.youtube.com/watch?v=0eqiQNPbe8I',
'https://www.youtube.com/watch?v=DYZXrzEvoIM',
'https://www.youtube.com/watch?v=9bIgZ_ILn6I',
'https://www.youtube.com/watch?v=TM8KRHnTI_M',
'https://www.youtube.com/watch?v=HfG5IyMc89w',
'https://www.youtube.com/watch?v=YbYMVyuuRvY',
'https://www.youtube.com/watch?v=V0pthiN2Clc',
'https://www.youtube.com/watch?v=LTPuJ541N0o',
'https://www.youtube.com/watch?v=O4L8f2GI1NM',
'https://www.youtube.com/watch?v=UH0v9pjo-I0',
'https://www.youtube.com/watch?v=efRsPcDuHmE',
'https://www.youtube.com/watch?v=g-HfLkiZsYg',
'https://www.youtube.com/watch?v=hCdGvWuF1d8',
'https://www.youtube.com/watch?v=Iaz6JIxDh6Y',
'https://www.youtube.com/watch?v=mn_MaruQIfo',
'https://www.youtube.com/watch?v=0bNIwlODieI',
'https://www.youtube.com/watch?v=jnZUHQHGTYM',
'https://www.youtube.com/watch?v=uiCeqySVhCE',
'https://www.youtube.com/watch?v=dd96_vUNsYQ',
'https://www.youtube.com/watch?v=YlUfufBj5vU',
'https://www.youtube.com/watch?v=X6mAf_uPy-o',
'https://www.youtube.com/watch?v=8LqzVjev4vg',
'https://www.youtube.com/watch?v=zRvqkUoiKJo',
'https://www.youtube.com/watch?v=URZihNBsnjA',
'https://www.youtube.com/watch?v=dg5ciDyauI4',
'https://www.youtube.com/watch?v=4plIX68mdE8',
'https://www.youtube.com/watch?v=ba_cJYpONvw',
'https://www.youtube.com/watch?v=SVk3Nh_seq0',
'https://www.youtube.com/watch?v=8ZWS0A5m8f8',
'https://www.youtube.com/watch?v=TZeCBVvH3bU',
'https://www.youtube.com/watch?v=WBSi93dzxPA',
'https://www.youtube.com/watch?v=OWhbSKlN3fE',
'https://www.youtube.com/watch?v=YvcCyRGOIYw',
'https://www.youtube.com/watch?v=3lcDT_-3v2k',
'https://www.youtube.com/watch?v=TYAjo3DCY6M',
'https://www.youtube.com/watch?v=ZiPXFeL3KQU',
'https://www.youtube.com/watch?v=cr69xk1xiOU',
'https://www.youtube.com/watch?v=vq9irdLcZmU',
'https://www.youtube.com/watch?v=PHGuhEvdg6M',
'https://www.youtube.com/watch?v=jnc1Ay6X1bg',
'https://www.youtube.com/watch?v=hweZuG-UBmM',
'https://www.youtube.com/watch?v=EXBNGlUVgFs',
'https://www.youtube.com/watch?v=YhSUknaEnec',
'https://www.youtube.com/watch?v=W7QBt61ChR4',
'https://www.youtube.com/watch?v=kVNq8knHGew',
'https://www.youtube.com/watch?v=F5pJR45HNEM',
'https://www.youtube.com/watch?v=D5in5EdjhD0',
'https://www.youtube.com/watch?v=rV2KFnsfYPk',
'https://www.youtube.com/watch?v=DFaNajmJ2oc',
'https://www.youtube.com/watch?v=RMxKc-uFEnU',
'https://www.youtube.com/watch?v=1pH8gCEZ4Ds',
'https://www.youtube.com/watch?v=oUUkwmTmYqM',
'https://www.youtube.com/watch?v=CHS1NraVsAc',
'https://www.youtube.com/watch?v=U7EyfO0TRm4',
'https://www.youtube.com/watch?v=0XnihgXhh3c',
'https://www.youtube.com/watch?v=9DvmLMUfGss',
'https://www.youtube.com/watch?v=XNSxj0TVeJs',
'https://www.youtube.com/watch?v=-U56oPQado8',
'https://www.youtube.com/watch?v=2qCaW1_4LBQ',
'https://www.youtube.com/watch?v=uEGVwVaauDM',
'https://www.youtube.com/watch?v=2G6kf7XM9Nk',
'https://www.youtube.com/watch?v=2HvGy2gY0eM',
'https://www.youtube.com/watch?v=Grn2EZxcUeA',
'https://www.youtube.com/watch?v=-tU918oK85o',
'https://www.youtube.com/watch?v=Y3PwG4UoJ0Y',
'https://www.youtube.com/watch?v=h00PgJSL8wY',
'https://www.youtube.com/watch?v=8NTWv4qr3iA',
'https://www.youtube.com/watch?v=StGGngQIwY0',
'https://www.youtube.com/watch?v=L7ZK8J69CW0',
'https://www.youtube.com/watch?v=PHEKSSpOZGc',
'https://www.youtube.com/watch?v=2yVQOCzw34U',
'https://www.youtube.com/watch?v=5adE7kHqTLU',
'https://www.youtube.com/watch?v=guPlS0___ww',
'https://www.youtube.com/watch?v=Ah5RsGc5Tnk',
'https://www.youtube.com/watch?v=86RY40TvRgE',
'https://www.youtube.com/watch?v=-NGQ9INMnq0',
'https://www.youtube.com/watch?v=szIGZVrSAyc',
'https://www.youtube.com/watch?v=CvZRsdHgxgA',
'https://www.youtube.com/watch?v=mk8pxyAWTBk',
'https://www.youtube.com/watch?v=KYlyb1Bx9Ic',
'https://www.youtube.com/watch?v=8-jh2R-_eQY',
'https://www.youtube.com/watch?v=MXh1_ubCQAI',
'https://www.youtube.com/watch?v=Zh7ps1KWl_M',
'https://www.youtube.com/watch?v=uXPvPxOFba0',
'https://www.youtube.com/watch?v=-3kAld2YwgU',
'https://www.youtube.com/watch?v=nvo9O4A18d0',
'https://www.youtube.com/watch?v=5OA-KgruEEg',
'https://www.youtube.com/watch?v=fy69-u4rcHU',
'https://www.youtube.com/watch?v=4vGc0xdZsf4',
'https://www.youtube.com/watch?v=K_Gym829MIM',
'https://www.youtube.com/watch?v=GL1GcRrbPm0',
'https://www.youtube.com/watch?v=e8TihISaf7E',
'https://www.youtube.com/watch?v=9JScy7ulDpE',
'https://www.youtube.com/watch?v=TAP0xk-c4mk',
'https://www.youtube.com/watch?v=u0t34c3W8d0',
'https://www.youtube.com/watch?v=e1JWr03P9W8',
'https://www.youtube.com/watch?v=H-fRSrfr2L0',
'https://www.youtube.com/watch?v=tKFgMHFXaYs',
'https://www.youtube.com/watch?v=6gD-W-X6hdc',
'https://www.youtube.com/watch?v=WpJjuotD534',
'https://www.youtube.com/watch?v=VBI9mC77igo',
'https://www.youtube.com/watch?v=WSN_8YGX6W8',
'https://www.youtube.com/watch?v=QKq38COoTG8',
'https://www.youtube.com/watch?v=bgqwfCyZpao',
'https://www.youtube.com/watch?v=7TLZN92-dZo',
'https://www.youtube.com/watch?v=VoRmqbPFasE',
'https://www.youtube.com/watch?v=HDXmWCzw5_U'
]
return urls_yt_add
def urls_yt_remove():
urls_yt_remove = [
# The Forest Revealed
'https://www.youtube.com/watch?v=qAAX4olzVuQ',
'https://www.youtube.com/watch?v=KafxQqt-Ntw',
'https://www.youtube.com/watch?v=2s70bDIMs8c',
'https://www.youtube.com/watch?v=p4y8fTl00tI',
'https://www.youtube.com/watch?v=MKxXyt-YTss',
'https://www.youtube.com/watch?v=_mpZplkvm-s',
'https://www.youtube.com/watch?v=cOVl0ub0Mls',
'https://www.youtube.com/watch?v=kejMQ0tcdhs',
'https://www.youtube.com/watch?v=8x6uvAH0Yr8',
'https://www.youtube.com/watch?v=0M1G5DLnmfA',
'https://www.youtube.com/watch?v=0kpaVxnbWh4',
'https://www.youtube.com/watch?v=96Zt9l53xWk',
'https://www.youtube.com/watch?v=R6dZ_obn1NU',
'https://www.youtube.com/watch?v=38uAAYHGPYo',
'https://www.youtube.com/watch?v=yJzCNfdTf60',
'https://www.youtube.com/watch?v=7eNEfeIytMs',
'https://www.youtube.com/watch?v=OGQZtnyi4hc',
'https://www.youtube.com/watch?v=E6vLshKTOxQ',
'https://www.youtube.com/watch?v=-bLb7VZnqko',
# Ricardo Ruiz
'https://www.youtube.com/watch?v=AUniGUu5hGE',
'https://www.youtube.com/watch?v=CooK_RWMreI',
'https://www.youtube.com/watch?v=05XGb2Aw5lA',
'https://www.youtube.com/watch?v=SDZYRvwI7pY',
'https://www.youtube.com/watch?v=VZYNRKKuaXY',
# Chomskyan
'https://www.youtube.com/watch?v=HWgv0Pr1i9o',
'https://www.youtube.com/watch?v=7-iGi4H2Vqs',
'https://www.youtube.com/watch?v=UTnd8jG9Lfo',
'https://www.youtube.com/watch?v=eYr2wcsNGVc',
'https://www.youtube.com/watch?v=a27mnDbHCow',
'https://www.youtube.com/watch?v=4tfkiUtB3_0',
'https://www.youtube.com/watch?v=QNLHDsZZ50Q',
'https://www.youtube.com/watch?v=zIERifyW_aI',
# Linguistics
'https://www.youtube.com/watch?v=ycC5oZqNXsk',
'https://www.youtube.com/watch?v=yRSxR9Us9QM',
'https://www.youtube.com/watch?v=msOnzHPeXzE',
'https://www.youtube.com/watch?v=24PcF7LDQ7M',
# ...XCGIY
'https://www.youtube.com/watch?v=0lupJCD8DEw',
# ...0xylW
'https://www.youtube.com/watch?v=O4GLzz-9lRY',
'https://www.youtube.com/watch?v=6-d23ma3qz8',
'https://www.youtube.com/watch?v=XUBYI97cUgU',
'https://www.youtube.com/watch?v=ww__v9i-PU4',
'https://www.youtube.com/watch?v=lWyuKzH9NSQ',
'https://www.youtube.com/watch?v=IyRoVOSVqZ8',
# Unknown
'https://www.youtube.com/watch?v=wlrOyum4US0',
'https://www.youtube.com/watch?v=34LGPIXvU5M'
]
return urls_yt_remove
# Remove duplicates from across YT channels/playlists/videos
# by using youtube-dl to create lists of video ids
def urls_yt():
yt_add_list = []
for url in urls_yt_add():
output = subprocess.check_output('powershell.exe ' + './youtube-dl -j --flat-playlist ' + url, stderr=subprocess.STDOUT, shell=True)
for jsonObj in output.decode('UTF-8').splitlines():
yt_add_dict = json.loads(jsonObj)
yt_add_list.append(yt_add_dict["id"])
yt_add_set = set(yt_add_list)
yt_remove_list = []
for url in urls_yt_remove():
output = subprocess.check_output('powershell.exe ' + './youtube-dl -j --flat-playlist ' + url, stderr=subprocess.STDOUT, shell=True)
for jsonObj in output.decode('UTF-8').splitlines():
yt_remove_dict = json.loads(jsonObj)
yt_remove_list.append(yt_remove_dict["id"])
yt_remove_set = set(yt_remove_list)
return yt_add_set.difference(yt_remove_set)
Reference in a new issue View git blame Copy permalink