prehistoric-systems/the-chomsky-index
1
0
Fork 0
mirror of https://github.com/jasons-gh/the-chomsky-index.git synced 2026-03-11 08:54:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
the-chomsky-index/sources.py
Jason Lobo 901ef2dd97 Update sources.py
2021-08-05 22:26:19 +01:00

496 lines
23 KiB
Python
Raw Permalink Blame History

import youtube_dl
from bs4 import BeautifulSoup
from urllib.request import Request, urlopen
def urls_yt_add():
urls_yt_add = [
# Channels
# The Forest Revealed
'https://www.youtube.com/playlist?list=UU6Od0Wqrl8PH9pU0s2kU9cw',
# Chomsky's Philosophy
'https://www.youtube.com/playlist?list=UUHuLYgw4dGbC2BuZQqPWV1g',
# Ricardo Ruiz
'https://www.youtube.com/playlist?list=UUXZXj5KotSzQBQDKdK-o49Q',
# Chomskyan
'https://www.youtube.com/playlist?list=UU7iZ-JXZqKMAi3_g0n41Mbw',
# letNOAMspeak
'https://www.youtube.com/playlist?list=UU_QSlX5r6uky8evEIihytyw',
# noamychomskyy
'https://www.youtube.com/playlist?list=UUSvQGNETA0mImQsxixw_RaQ',
# Carlos Garcia
'https://www.youtube.com/playlist?list=UUmYouQK9gYJOJ_-rz7aXnXg',
# Ramin Zareian
'https://www.youtube.com/playlist?list=UU7Kki7JHZvmYcxNWI2-Dqjg',
# Playlists
# Chomsky Archive
'https://www.youtube.com/playlist?list=PLJAP0acmX6BmeJ7Yktv9kmzlJ-dLHdD1h',
# Cognitive Revolution
'https://www.youtube.com/playlist?list=PL6nOdowQ4jwmJAAtK5ixnl5v4EoBqzMhS',
# Misc
'https://www.youtube.com/playlist?list=PL13E62E4DC420B97A',
# Nationalism
'https://www.youtube.com/playlist?list=PL2DT1k6e9kKX_cC8WEx369U9hSAiAUNGh',
# Democracy Now!
'https://www.youtube.com/playlist?list=PL8E296CDF21DE9D35',
# Linguistics
'https://www.youtube.com/playlist?list=PLHU1ryHGd_xaSQfxOmaqf-oueT2WO5aQE',
# infiniteinfiniteinfi
'https://www.youtube.com/playlist?list=PL8B20BDCE43D7A2BA',
# Misc
'https://www.youtube.com/playlist?list=PLi6hnQx6eEsARtpAp7BPOruiNnE2XCGIY',
# Misc
'https://www.youtube.com/playlist?list=PL528C06FF92A3DFF0',
# Misc
'https://www.youtube.com/playlist?list=PLi6hnQx6eEsCvp_SR50-uK_QD2YU0xylW',
# Misc
'https://www.youtube.com/playlist?list=PLjRJEmwDWKwrC4qLrfZP5k6aY7ui0p0eJ',
# Chomsky Sessions
'https://www.youtube.com/playlist?list=PLn5kIRhssBsn6CqX4w7fhWozb5oUwu1ap',
# Misc
'https://www.youtube.com/playlist?list=PLbfLNPEPw2rKwaQAn_h9DB5pHZmofFhh-',
# Boston College
'https://www.youtube.com/playlist?list=PLDAC3BFD45C357BF1',
# Videos
'https://www.youtube.com/watch?v=hrhFI1rsDDE',
'https://www.youtube.com/watch?v=oJSQ7_I7zmw',
'https://www.youtube.com/watch?v=r514RhgISv0',
'https://www.youtube.com/watch?v=GPHew_smDjY',
'https://www.youtube.com/watch?v=Lv4QSpY3cbY',
'https://www.youtube.com/watch?v=fHvhWZJMPwc',
'https://www.youtube.com/watch?v=hrhFI1rsDDE',
'https://www.youtube.com/watch?v=6XZ29-kGCmg',
'https://www.youtube.com/watch?v=kfvtD4ptZ-k',
'https://www.youtube.com/watch?v=qDjXsdslrYc',
'https://www.youtube.com/watch?v=XeWWz4y1coU',
'https://www.youtube.com/watch?v=apfgfG-Gmp4',
'https://www.youtube.com/watch?v=Vmh10WtC7ws',
'https://www.youtube.com/watch?v=_AMzzKonrTQ',
'https://www.youtube.com/watch?v=owp9ooAbeXc',
'https://www.youtube.com/watch?v=EA7Dj-dmzEE',
'https://www.youtube.com/watch?v=DJKf9wKSbZ4',
'https://www.youtube.com/watch?v=cMscNuSUy0I',
'https://www.youtube.com/watch?v=8ED1Xt3IoRc',
'https://www.youtube.com/watch?v=yuVqfKYbGvE',
'https://www.youtube.com/watch?v=SL2IpMNTz1A',
'https://www.youtube.com/watch?v=mpIF7xjGyRs',
'https://www.youtube.com/watch?v=xebTB_Lh8I8',
'https://www.youtube.com/watch?v=xgYbM2Q9cso',
'https://www.youtube.com/watch?v=WGdljrhdSKA',
'https://www.youtube.com/watch?v=Ud96nzm5Dvk',
'https://www.youtube.com/watch?v=9B_eLoVq4yI',
'https://www.youtube.com/watch?v=KZthDGH6WCU',
'https://www.youtube.com/watch?v=xOVuhgx0A0E',
'https://www.youtube.com/watch?v=XXM4Py1UA3g',
'https://www.youtube.com/watch?v=widzhMeoAy0',
'https://www.youtube.com/watch?v=ea1p1eevcfo',
'https://www.youtube.com/watch?v=t-N3In2rLI4',
'https://www.youtube.com/watch?v=8wb6LFOYyQQ',
'https://www.youtube.com/watch?v=kP9pUqqeUD0',
'https://www.youtube.com/watch?v=OEdfJP9S4NE',
'https://www.youtube.com/watch?v=rQLq9iLQseA',
'https://www.youtube.com/watch?v=A5MQw3N53MY',
'https://www.youtube.com/watch?v=2oqQgQYBWvI',
'https://www.youtube.com/watch?v=8TNOq7zgBZ8',
'https://www.youtube.com/watch?v=VJFbxe21y1M',
'https://www.youtube.com/watch?v=unWjIwBarIY',
'https://www.youtube.com/watch?v=kwN5BV8vZUg',
'https://www.youtube.com/watch?v=b2VyTIMhUqw',
'https://www.youtube.com/watch?v=d-6EXZ7yM2E',
'https://www.youtube.com/watch?v=39902cn5lX8',
'https://www.youtube.com/watch?v=lciwaaFQqCk',
'https://www.youtube.com/watch?v=gP6T_-bqt40',
'https://www.youtube.com/watch?v=MHhgCFJWgM0',
'https://www.youtube.com/watch?v=w-wSasUe1hc',
'https://www.youtube.com/watch?v=BW0l2ESybiY',
'https://www.youtube.com/watch?v=1_dBnu4MUIQ',
'https://www.youtube.com/watch?v=twUinwhHL-k',
'https://www.youtube.com/watch?v=zi6ae6kZNqE',
'https://www.youtube.com/watch?v=Vq6IgBMg3lI',
'https://www.youtube.com/watch?v=k1kqgTKf0kw',
'https://www.youtube.com/watch?v=hX3ZbF0i2aE',
'https://www.youtube.com/watch?v=HO0OpQN9d1s',
'https://www.youtube.com/watch?v=VLwjqrfRAYM',
'https://www.youtube.com/watch?v=Rtt3d0mtJe0',
'https://www.youtube.com/watch?v=SO3tE7clWhI',
'https://www.youtube.com/watch?v=ZXfXy771K8w',
'https://www.youtube.com/watch?v=VUYYZdE5Dmk',
'https://www.youtube.com/watch?v=j7B6vVv57aw',
'https://www.youtube.com/watch?v=Rye4tVkm4iA',
'https://www.youtube.com/watch?v=C9tw09IzQdM',
'https://www.youtube.com/watch?v=4MM55B0zMB4',
'https://www.youtube.com/watch?v=O2dyYT3SdP8',
'https://www.youtube.com/watch?v=heGfKTRbWfo',
'https://www.youtube.com/watch?v=LQOeDg0qFf4',
'https://www.youtube.com/watch?v=mFeCq-KZ9nM',
'https://www.youtube.com/watch?v=n_LD6alcsn0',
'https://www.youtube.com/watch?v=RiTl0moYYeQ',
'https://www.youtube.com/watch?v=jgPyTXw65l0',
'https://www.youtube.com/watch?v=Jr8f6f0cnuc',
'https://www.youtube.com/watch?v=6UYbGA6yB7Q',
'https://www.youtube.com/watch?v=sb_TM5uUoSM',
'https://www.youtube.com/watch?v=SZBiZHpK398',
'https://www.youtube.com/watch?v=IqfWUe0UbZA',
'https://www.youtube.com/watch?v=vaF-W2Wg0Pg',
'https://www.youtube.com/watch?v=MER4y8AMjrc',
'https://www.youtube.com/watch?v=aR56qEj1u58',
'https://www.youtube.com/watch?v=0eqiQNPbe8I',
'https://www.youtube.com/watch?v=DYZXrzEvoIM',
'https://www.youtube.com/watch?v=9bIgZ_ILn6I',
'https://www.youtube.com/watch?v=TM8KRHnTI_M',
'https://www.youtube.com/watch?v=HfG5IyMc89w',
'https://www.youtube.com/watch?v=YbYMVyuuRvY',
'https://www.youtube.com/watch?v=V0pthiN2Clc',
'https://www.youtube.com/watch?v=LTPuJ541N0o',
'https://www.youtube.com/watch?v=O4L8f2GI1NM',
'https://www.youtube.com/watch?v=UH0v9pjo-I0',
'https://www.youtube.com/watch?v=efRsPcDuHmE',
'https://www.youtube.com/watch?v=g-HfLkiZsYg',
'https://www.youtube.com/watch?v=hCdGvWuF1d8',
'https://www.youtube.com/watch?v=Iaz6JIxDh6Y',
'https://www.youtube.com/watch?v=mn_MaruQIfo',
'https://www.youtube.com/watch?v=0bNIwlODieI',
'https://www.youtube.com/watch?v=jnZUHQHGTYM',
'https://www.youtube.com/watch?v=uiCeqySVhCE',
'https://www.youtube.com/watch?v=dd96_vUNsYQ',
'https://www.youtube.com/watch?v=YlUfufBj5vU',
'https://www.youtube.com/watch?v=X6mAf_uPy-o',
'https://www.youtube.com/watch?v=8LqzVjev4vg',
'https://www.youtube.com/watch?v=zRvqkUoiKJo',
'https://www.youtube.com/watch?v=URZihNBsnjA',
'https://www.youtube.com/watch?v=dg5ciDyauI4',
'https://www.youtube.com/watch?v=4plIX68mdE8',
'https://www.youtube.com/watch?v=ba_cJYpONvw',
'https://www.youtube.com/watch?v=SVk3Nh_seq0',
'https://www.youtube.com/watch?v=8ZWS0A5m8f8',
'https://www.youtube.com/watch?v=TZeCBVvH3bU',
'https://www.youtube.com/watch?v=WBSi93dzxPA',
'https://www.youtube.com/watch?v=OWhbSKlN3fE',
'https://www.youtube.com/watch?v=YvcCyRGOIYw',
'https://www.youtube.com/watch?v=3lcDT_-3v2k',
'https://www.youtube.com/watch?v=TYAjo3DCY6M',
'https://www.youtube.com/watch?v=ZiPXFeL3KQU',
'https://www.youtube.com/watch?v=cr69xk1xiOU',
'https://www.youtube.com/watch?v=omwcM_uesLs',
'https://www.youtube.com/watch?v=fkcGjy2wenk',
'https://www.youtube.com/watch?v=zTlSmQKqS-Q',
'https://www.youtube.com/watch?v=6YxPOGC2K8A',
'https://www.youtube.com/watch?v=3vpOmbGsLng',
'https://www.youtube.com/watch?v=E-yeVDKkRsM',
'https://www.youtube.com/watch?v=zIrIBP-BalQ',
'https://www.youtube.com/watch?v=d8Wrwo2a1dg',
'https://www.youtube.com/watch?v=WjIZEIDxFGc',
'https://www.youtube.com/watch?v=XhUOUb9mXek',
'https://www.youtube.com/watch?v=LNzQESt7g-0',
'https://www.youtube.com/watch?v=LmolEKym3rs',
'https://www.youtube.com/watch?v=xikoTE3zyME',
'https://www.youtube.com/watch?v=IEcJtk4-uZE',
'https://www.youtube.com/watch?v=apfgfG-Gmp4',
'https://www.youtube.com/watch?v=EA7Dj-dmzEE',
'https://www.youtube.com/watch?v=cMscNuSUy0I',
'https://www.youtube.com/watch?v=SL2IpMNTz1A',
'https://www.youtube.com/watch?v=mpIF7xjGyRs',
'https://www.youtube.com/watch?v=xebTB_Lh8I8',
'https://www.youtube.com/watch?v=WGdljrhdSKA',
'https://www.youtube.com/watch?v=9B_eLoVq4yI',
'https://www.youtube.com/watch?v=ea1p1eevcfo',
'https://www.youtube.com/watch?v=t-N3In2rLI4',
'https://www.youtube.com/watch?v=8wb6LFOYyQQ',
'https://www.youtube.com/watch?v=rQLq9iLQseA',
'https://www.youtube.com/watch?v=Vmh10WtC7ws',
'https://www.youtube.com/watch?v=A5MQw3N53MY',
'https://www.youtube.com/watch?v=d-6EXZ7yM2E',
'https://www.youtube.com/watch?v=39902cn5lX8',
'https://www.youtube.com/watch?v=lciwaaFQqCk',
'https://www.youtube.com/watch?v=w-wSasUe1hc',
'https://www.youtube.com/watch?v=zi6ae6kZNqE',
'https://www.youtube.com/watch?v=NaxkTCUtzX0',
'https://www.youtube.com/watch?v=widzhMeoAy0',
'https://www.youtube.com/watch?v=KZthDGH6WCU',
'https://www.youtube.com/watch?v=xOVuhgx0A0E',
'https://www.youtube.com/watch?v=nmguZTONFK4',
'https://www.youtube.com/watch?v=FUI71dZJ43k',
'https://www.youtube.com/watch?v=Fc0WAkPq_JU',
'https://www.youtube.com/watch?v=q0--w3_JlR4',
'https://www.youtube.com/watch?v=nV9vCx_vCVM',
'https://www.youtube.com/watch?v=DyEKsL-NSmw',
'https://www.youtube.com/watch?v=QsIBauyOdLk',
'https://www.youtube.com/watch?v=iQSdF5cs2YE',
'https://www.youtube.com/watch?v=lQ8JX_64E7Q',
'https://www.youtube.com/watch?v=dgt652020RY',
'https://www.youtube.com/watch?v=wUPZ8rSESZo',
'https://www.youtube.com/watch?v=HNZD0B1i1zM',
'https://www.youtube.com/watch?v=kP9pUqqeUD0',
'https://www.youtube.com/watch?v=OEdfJP9S4NE',
'https://www.youtube.com/watch?v=RYCS7LKu1UA',
'https://www.youtube.com/watch?v=yN5ESpYyXmY',
'https://www.youtube.com/watch?v=cdEZrvS_nnY',
'https://www.youtube.com/watch?v=ksUwluZXyWc',
'https://www.youtube.com/watch?v=TgbPQNjaSno',
'https://www.youtube.com/watch?v=1u2dwfRb9S8',
'https://www.youtube.com/watch?v=zZc5Z9C7UfU',
'https://www.youtube.com/watch?v=2xsaa6sN0cM',
'https://www.youtube.com/watch?v=wQJOljq3SZk',
'https://www.youtube.com/watch?v=Ro-kkxkHHuU',
'https://www.youtube.com/watch?v=XwQ-oZY4oKA',
'https://www.youtube.com/watch?v=Mk1U8MAudbk',
'https://www.youtube.com/watch?v=vq9irdLcZmU',
'https://www.youtube.com/watch?v=PHGuhEvdg6M',
'https://www.youtube.com/watch?v=jnc1Ay6X1bg',
'https://www.youtube.com/watch?v=hweZuG-UBmM',
'https://www.youtube.com/watch?v=EXBNGlUVgFs',
'https://www.youtube.com/watch?v=YhSUknaEnec',
'https://www.youtube.com/watch?v=W7QBt61ChR4',
'https://www.youtube.com/watch?v=kVNq8knHGew',
'https://www.youtube.com/watch?v=F5pJR45HNEM',
'https://www.youtube.com/watch?v=D5in5EdjhD0',
'https://www.youtube.com/watch?v=rV2KFnsfYPk',
'https://www.youtube.com/watch?v=DFaNajmJ2oc',
'https://www.youtube.com/watch?v=RMxKc-uFEnU',
'https://www.youtube.com/watch?v=1pH8gCEZ4Ds',
'https://www.youtube.com/watch?v=oUUkwmTmYqM',
'https://www.youtube.com/watch?v=CHS1NraVsAc',
'https://www.youtube.com/watch?v=U7EyfO0TRm4',
'https://www.youtube.com/watch?v=0XnihgXhh3c',
'https://www.youtube.com/watch?v=9DvmLMUfGss',
'https://www.youtube.com/watch?v=XNSxj0TVeJs',
'https://www.youtube.com/watch?v=-U56oPQado8',
'https://www.youtube.com/watch?v=2qCaW1_4LBQ',
'https://www.youtube.com/watch?v=uEGVwVaauDM',
'https://www.youtube.com/watch?v=2G6kf7XM9Nk',
'https://www.youtube.com/watch?v=2HvGy2gY0eM',
'https://www.youtube.com/watch?v=Grn2EZxcUeA',
'https://www.youtube.com/watch?v=-tU918oK85o',
'https://www.youtube.com/watch?v=Y3PwG4UoJ0Y',
'https://www.youtube.com/watch?v=h00PgJSL8wY',
'https://www.youtube.com/watch?v=8NTWv4qr3iA',
'https://www.youtube.com/watch?v=StGGngQIwY0',
'https://www.youtube.com/watch?v=L7ZK8J69CW0',
'https://www.youtube.com/watch?v=PHEKSSpOZGc',
'https://www.youtube.com/watch?v=2yVQOCzw34U',
'https://www.youtube.com/watch?v=5adE7kHqTLU',
'https://www.youtube.com/watch?v=guPlS0___ww',
'https://www.youtube.com/watch?v=Ah5RsGc5Tnk',
'https://www.youtube.com/watch?v=86RY40TvRgE',
'https://www.youtube.com/watch?v=-NGQ9INMnq0',
'https://www.youtube.com/watch?v=szIGZVrSAyc',
'https://www.youtube.com/watch?v=CvZRsdHgxgA',
'https://www.youtube.com/watch?v=mk8pxyAWTBk',
'https://www.youtube.com/watch?v=KYlyb1Bx9Ic',
'https://www.youtube.com/watch?v=8-jh2R-_eQY',
'https://www.youtube.com/watch?v=MXh1_ubCQAI',
'https://www.youtube.com/watch?v=Zh7ps1KWl_M',
'https://www.youtube.com/watch?v=uXPvPxOFba0',
'https://www.youtube.com/watch?v=-3kAld2YwgU',
'https://www.youtube.com/watch?v=nvo9O4A18d0',
'https://www.youtube.com/watch?v=5OA-KgruEEg',
'https://www.youtube.com/watch?v=fy69-u4rcHU',
'https://www.youtube.com/watch?v=4vGc0xdZsf4',
'https://www.youtube.com/watch?v=K_Gym829MIM',
'https://www.youtube.com/watch?v=GL1GcRrbPm0',
'https://www.youtube.com/watch?v=e8TihISaf7E',
'https://www.youtube.com/watch?v=9JScy7ulDpE',
'https://www.youtube.com/watch?v=TAP0xk-c4mk',
'https://www.youtube.com/watch?v=u0t34c3W8d0',
'https://www.youtube.com/watch?v=e1JWr03P9W8',
'https://www.youtube.com/watch?v=H-fRSrfr2L0',
'https://www.youtube.com/watch?v=tKFgMHFXaYs',
'https://www.youtube.com/watch?v=6gD-W-X6hdc',
'https://www.youtube.com/watch?v=WpJjuotD534',
'https://www.youtube.com/watch?v=VBI9mC77igo',
'https://www.youtube.com/watch?v=WSN_8YGX6W8',
'https://www.youtube.com/watch?v=QKq38COoTG8',
'https://www.youtube.com/watch?v=bgqwfCyZpao',
'https://www.youtube.com/watch?v=7TLZN92-dZo',
'https://www.youtube.com/watch?v=VoRmqbPFasE',
'https://www.youtube.com/watch?v=HDXmWCzw5_U',
'https://www.youtube.com/watch?v=1OGIJE8AzqM',
'https://www.youtube.com/watch?v=nv1eBf58JfI',
'https://www.youtube.com/watch?v=b-u9f5IJa1s',
'https://www.youtube.com/watch?v=lWGhJ63OXxM',
'https://www.youtube.com/watch?v=46bH_jV34ig',
'https://www.youtube.com/watch?v=hLsNOEfDrNw',
'https://www.youtube.com/watch?v=rH8SicnqSC4',
'https://www.youtube.com/watch?v=Oxr4KJtJgPc',
####
'https://www.youtube.com/watch?v=E-oqU5BPvI8',
]
return urls_yt_add
def urls_yt_remove():
urls_yt_remove = [
# The Forest Revealed
'https://www.youtube.com/watch?v=qAAX4olzVuQ',
'https://www.youtube.com/watch?v=KafxQqt-Ntw',
'https://www.youtube.com/watch?v=2s70bDIMs8c',
'https://www.youtube.com/watch?v=p4y8fTl00tI',
'https://www.youtube.com/watch?v=MKxXyt-YTss',
'https://www.youtube.com/watch?v=_mpZplkvm-s',
'https://www.youtube.com/watch?v=cOVl0ub0Mls',
'https://www.youtube.com/watch?v=kejMQ0tcdhs',
'https://www.youtube.com/watch?v=8x6uvAH0Yr8',
'https://www.youtube.com/watch?v=0M1G5DLnmfA',
'https://www.youtube.com/watch?v=0kpaVxnbWh4',
'https://www.youtube.com/watch?v=96Zt9l53xWk',
'https://www.youtube.com/watch?v=R6dZ_obn1NU',
'https://www.youtube.com/watch?v=38uAAYHGPYo',
'https://www.youtube.com/watch?v=yJzCNfdTf60',
'https://www.youtube.com/watch?v=7eNEfeIytMs',
'https://www.youtube.com/watch?v=OGQZtnyi4hc',
'https://www.youtube.com/watch?v=E6vLshKTOxQ',
'https://www.youtube.com/watch?v=-bLb7VZnqko',
# Ricardo Ruiz
'https://www.youtube.com/watch?v=AUniGUu5hGE',
'https://www.youtube.com/watch?v=CooK_RWMreI',
'https://www.youtube.com/watch?v=05XGb2Aw5lA',
'https://www.youtube.com/watch?v=SDZYRvwI7pY',
'https://www.youtube.com/watch?v=VZYNRKKuaXY',
# Chomskyan
'https://www.youtube.com/watch?v=HWgv0Pr1i9o',
'https://www.youtube.com/watch?v=7-iGi4H2Vqs',
'https://www.youtube.com/watch?v=UTnd8jG9Lfo',
'https://www.youtube.com/watch?v=eYr2wcsNGVc',
'https://www.youtube.com/watch?v=a27mnDbHCow',
'https://www.youtube.com/watch?v=4tfkiUtB3_0',
'https://www.youtube.com/watch?v=QNLHDsZZ50Q',
'https://www.youtube.com/watch?v=zIERifyW_aI',
# Linguistics
'https://www.youtube.com/watch?v=ycC5oZqNXsk',
'https://www.youtube.com/watch?v=yRSxR9Us9QM',
'https://www.youtube.com/watch?v=msOnzHPeXzE',
'https://www.youtube.com/watch?v=24PcF7LDQ7M',
# ...XCGIY
'https://www.youtube.com/watch?v=0lupJCD8DEw',
# ...0xylW
'https://www.youtube.com/watch?v=O4GLzz-9lRY',
'https://www.youtube.com/watch?v=6-d23ma3qz8',
'https://www.youtube.com/watch?v=XUBYI97cUgU',
'https://www.youtube.com/watch?v=ww__v9i-PU4',
'https://www.youtube.com/watch?v=lWyuKzH9NSQ',
'https://www.youtube.com/watch?v=IyRoVOSVqZ8',
# Ramin Zareian
'https://www.youtube.com/watch?v=U3sm-OUIDR4',
# Unknown
'https://www.youtube.com/watch?v=wlrOyum4US0',
'https://www.youtube.com/watch?v=34LGPIXvU5M',
'https://www.youtube.com/watch?v=oDK0kJSvRm8',
'https://www.youtube.com/watch?v=44h9QuWcJYk',
'https://www.youtube.com/watch?v=zi6ae6kZNqE',
'https://www.youtube.com/watch?v=hwNrHEq2vI0'
]
return urls_yt_remove
# Remove duplicates from across YT channels/playlists/videos
def urls_yt():
ydl = youtube_dl.YoutubeDL({'ignoreerrors': True,
'extract_flat': 'in_playlist',
'dump_single_json': True})
with ydl:
yt_add_set = set()
for url in urls_yt_add():
result = ydl.extract_info(url, download=False)
if result is None:
# Inaccessible video
continue
elif 'entries' in result:
# Playlist
yt_add_set.update(['https://www.youtube.com/watch?v=' + result_entry['url']
for result_entry in result['entries']
if result_entry is not None])
else:
# Video
yt_add_set.add(result['webpage_url'])
return yt_add_set.difference(urls_yt_remove())
def chomsky_dot_info_articles():
# Get articles
articles_url = "https://chomsky.info/articles/"
req = Request(articles_url, headers={'User-Agent': 'Mozilla/5.0'})
page = urlopen(req)
html = page.read().decode("utf-8")
soup_articles = BeautifulSoup(html, "html.parser")
urls = [a.get('href') for a in soup_articles.find_all('a')[23:-1]]
urls = [url.replace('http://', 'https://') for url in urls]
urls = ['https://chomsky.info/' + url[3:] if url.find('../') == 0 else url for url in urls]
urls = ['https://chomsky.info/' + url[2:] if url.find('..') == 0 else url for url in urls]
urls = [url.replace('zmag.org', 'zcomm.org') for url in urls]
urls.remove('https://www.journals.uchicago.edu/doi/abs/10.1086/291699')
urls.remove('https://www.chomsky.info/articles/200501--.pdf')
urls.remove('https://www.chomsky.info/articles/200310--.pdf')
urls.remove('https://www.chomsky.info/articles/20021122.pdf')
urls.remove('https://www.chomsky.info/articles/2000----.pdf')
urls.remove('https://www.chomsky.info/articles/1986----.pdf')
return set(urls)
def chomsky_dot_info_interviews():
# Get interviews
interviews_url = "https://chomsky.info/interviews/"
req = Request(interviews_url, headers={'User-Agent': 'Mozilla/5.0'})
page = urlopen(req)
html = page.read().decode("utf-8")
soup_articles = BeautifulSoup(html, "html.parser")
urls = [a.get('href') for a in soup_articles.find_all('a')[23:-1]]
urls = [url.replace('http://', 'https://') for url in urls]
urls = ['https://chomsky.info/' + url[3:] if url.find('../') == 0 else url for url in urls]
urls = ['https://chomsky.info/' + url[2:] if url.find('..') == 0 else url for url in urls]
urls = [url.replace('zmag.org', 'zcomm.org') for url in urls]
urls.remove('https://www.petemccormack.com/social_005.htm')
urls.remove('https://smallbusiness.yahoo.com/geocities')
urls.remove('https://www.nbcnews.com/id/3080909')
return set(urls)
def other():
'https://theintercept.com/2019/10/31/deconstructed-special-the-noam-chomsky-interview/',
'https://couragefound.org/wp-content/uploads/2019/10/Letter-to-Permanent-Representatives.pdf',
return
def truthout_interviews():
'https://truthout.org/articles/noam-chomsky-democratic-party-centrism-risks-handing-election-to-trump/',
'https://truthout.org/articles/chomsky-without-us-aid-israel-wouldnt-be-killing-palestinians-en-masse/',
return
Reference in a new issue View git blame Copy permalink