import youtube_dl from bs4 import BeautifulSoup from urllib.request import Request, urlopen def urls_yt_add(): urls_yt_add = [ # Channels # The Forest Revealed 'https://www.youtube.com/playlist?list=UU6Od0Wqrl8PH9pU0s2kU9cw', # Chomsky's Philosophy 'https://www.youtube.com/playlist?list=UUHuLYgw4dGbC2BuZQqPWV1g', # Ricardo Ruiz 'https://www.youtube.com/playlist?list=UUXZXj5KotSzQBQDKdK-o49Q', # Chomskyan 'https://www.youtube.com/playlist?list=UU7iZ-JXZqKMAi3_g0n41Mbw', # letNOAMspeak 'https://www.youtube.com/playlist?list=UU_QSlX5r6uky8evEIihytyw', # noamychomskyy 'https://www.youtube.com/playlist?list=UUSvQGNETA0mImQsxixw_RaQ', # Carlos Garcia 'https://www.youtube.com/playlist?list=UUmYouQK9gYJOJ_-rz7aXnXg', # Ramin Zareian 'https://www.youtube.com/playlist?list=UU7Kki7JHZvmYcxNWI2-Dqjg', # Playlists # Chomsky Archive 'https://www.youtube.com/playlist?list=PLJAP0acmX6BmeJ7Yktv9kmzlJ-dLHdD1h', # Cognitive Revolution 'https://www.youtube.com/playlist?list=PL6nOdowQ4jwmJAAtK5ixnl5v4EoBqzMhS', # Misc 'https://www.youtube.com/playlist?list=PL13E62E4DC420B97A', # Nationalism 'https://www.youtube.com/playlist?list=PL2DT1k6e9kKX_cC8WEx369U9hSAiAUNGh', # Democracy Now! 'https://www.youtube.com/playlist?list=PL8E296CDF21DE9D35', # Linguistics 'https://www.youtube.com/playlist?list=PLHU1ryHGd_xaSQfxOmaqf-oueT2WO5aQE', # infiniteinfiniteinfi 'https://www.youtube.com/playlist?list=PL8B20BDCE43D7A2BA', # Misc 'https://www.youtube.com/playlist?list=PLi6hnQx6eEsARtpAp7BPOruiNnE2XCGIY', # Misc 'https://www.youtube.com/playlist?list=PL528C06FF92A3DFF0', # Misc 'https://www.youtube.com/playlist?list=PLi6hnQx6eEsCvp_SR50-uK_QD2YU0xylW', # Misc 'https://www.youtube.com/playlist?list=PLjRJEmwDWKwrC4qLrfZP5k6aY7ui0p0eJ', # Chomsky Sessions 'https://www.youtube.com/playlist?list=PLn5kIRhssBsn6CqX4w7fhWozb5oUwu1ap', # Misc 'https://www.youtube.com/playlist?list=PLbfLNPEPw2rKwaQAn_h9DB5pHZmofFhh-', # Boston College 'https://www.youtube.com/playlist?list=PLDAC3BFD45C357BF1', # Videos 'https://www.youtube.com/watch?v=hrhFI1rsDDE', 'https://www.youtube.com/watch?v=oJSQ7_I7zmw', 'https://www.youtube.com/watch?v=r514RhgISv0', 'https://www.youtube.com/watch?v=GPHew_smDjY', 'https://www.youtube.com/watch?v=Lv4QSpY3cbY', 'https://www.youtube.com/watch?v=fHvhWZJMPwc', 'https://www.youtube.com/watch?v=hrhFI1rsDDE', 'https://www.youtube.com/watch?v=6XZ29-kGCmg', 'https://www.youtube.com/watch?v=kfvtD4ptZ-k', 'https://www.youtube.com/watch?v=qDjXsdslrYc', 'https://www.youtube.com/watch?v=XeWWz4y1coU', 'https://www.youtube.com/watch?v=apfgfG-Gmp4', 'https://www.youtube.com/watch?v=Vmh10WtC7ws', 'https://www.youtube.com/watch?v=_AMzzKonrTQ', 'https://www.youtube.com/watch?v=owp9ooAbeXc', 'https://www.youtube.com/watch?v=EA7Dj-dmzEE', 'https://www.youtube.com/watch?v=DJKf9wKSbZ4', 'https://www.youtube.com/watch?v=cMscNuSUy0I', 'https://www.youtube.com/watch?v=8ED1Xt3IoRc', 'https://www.youtube.com/watch?v=yuVqfKYbGvE', 'https://www.youtube.com/watch?v=SL2IpMNTz1A', 'https://www.youtube.com/watch?v=mpIF7xjGyRs', 'https://www.youtube.com/watch?v=xebTB_Lh8I8', 'https://www.youtube.com/watch?v=xgYbM2Q9cso', 'https://www.youtube.com/watch?v=WGdljrhdSKA', 'https://www.youtube.com/watch?v=Ud96nzm5Dvk', 'https://www.youtube.com/watch?v=9B_eLoVq4yI', 'https://www.youtube.com/watch?v=KZthDGH6WCU', 'https://www.youtube.com/watch?v=xOVuhgx0A0E', 'https://www.youtube.com/watch?v=XXM4Py1UA3g', 'https://www.youtube.com/watch?v=widzhMeoAy0', 'https://www.youtube.com/watch?v=ea1p1eevcfo', 'https://www.youtube.com/watch?v=t-N3In2rLI4', 'https://www.youtube.com/watch?v=8wb6LFOYyQQ', 'https://www.youtube.com/watch?v=kP9pUqqeUD0', 'https://www.youtube.com/watch?v=OEdfJP9S4NE', 'https://www.youtube.com/watch?v=rQLq9iLQseA', 'https://www.youtube.com/watch?v=A5MQw3N53MY', 'https://www.youtube.com/watch?v=2oqQgQYBWvI', 'https://www.youtube.com/watch?v=8TNOq7zgBZ8', 'https://www.youtube.com/watch?v=VJFbxe21y1M', 'https://www.youtube.com/watch?v=unWjIwBarIY', 'https://www.youtube.com/watch?v=kwN5BV8vZUg', 'https://www.youtube.com/watch?v=b2VyTIMhUqw', 'https://www.youtube.com/watch?v=d-6EXZ7yM2E', 'https://www.youtube.com/watch?v=39902cn5lX8', 'https://www.youtube.com/watch?v=lciwaaFQqCk', 'https://www.youtube.com/watch?v=gP6T_-bqt40', 'https://www.youtube.com/watch?v=MHhgCFJWgM0', 'https://www.youtube.com/watch?v=w-wSasUe1hc', 'https://www.youtube.com/watch?v=BW0l2ESybiY', 'https://www.youtube.com/watch?v=1_dBnu4MUIQ', 'https://www.youtube.com/watch?v=twUinwhHL-k', 'https://www.youtube.com/watch?v=zi6ae6kZNqE', 'https://www.youtube.com/watch?v=Vq6IgBMg3lI', 'https://www.youtube.com/watch?v=k1kqgTKf0kw', 'https://www.youtube.com/watch?v=hX3ZbF0i2aE', 'https://www.youtube.com/watch?v=HO0OpQN9d1s', 'https://www.youtube.com/watch?v=VLwjqrfRAYM', 'https://www.youtube.com/watch?v=Rtt3d0mtJe0', 'https://www.youtube.com/watch?v=SO3tE7clWhI', 'https://www.youtube.com/watch?v=ZXfXy771K8w', 'https://www.youtube.com/watch?v=VUYYZdE5Dmk', 'https://www.youtube.com/watch?v=j7B6vVv57aw', 'https://www.youtube.com/watch?v=Rye4tVkm4iA', 'https://www.youtube.com/watch?v=C9tw09IzQdM', 'https://www.youtube.com/watch?v=4MM55B0zMB4', 'https://www.youtube.com/watch?v=O2dyYT3SdP8', 'https://www.youtube.com/watch?v=heGfKTRbWfo', 'https://www.youtube.com/watch?v=LQOeDg0qFf4', 'https://www.youtube.com/watch?v=mFeCq-KZ9nM', 'https://www.youtube.com/watch?v=n_LD6alcsn0', 'https://www.youtube.com/watch?v=RiTl0moYYeQ', 'https://www.youtube.com/watch?v=jgPyTXw65l0', 'https://www.youtube.com/watch?v=Jr8f6f0cnuc', 'https://www.youtube.com/watch?v=6UYbGA6yB7Q', 'https://www.youtube.com/watch?v=sb_TM5uUoSM', 'https://www.youtube.com/watch?v=SZBiZHpK398', 'https://www.youtube.com/watch?v=IqfWUe0UbZA', 'https://www.youtube.com/watch?v=vaF-W2Wg0Pg', 'https://www.youtube.com/watch?v=MER4y8AMjrc', 'https://www.youtube.com/watch?v=aR56qEj1u58', 'https://www.youtube.com/watch?v=0eqiQNPbe8I', 'https://www.youtube.com/watch?v=DYZXrzEvoIM', 'https://www.youtube.com/watch?v=9bIgZ_ILn6I', 'https://www.youtube.com/watch?v=TM8KRHnTI_M', 'https://www.youtube.com/watch?v=HfG5IyMc89w', 'https://www.youtube.com/watch?v=YbYMVyuuRvY', 'https://www.youtube.com/watch?v=V0pthiN2Clc', 'https://www.youtube.com/watch?v=LTPuJ541N0o', 'https://www.youtube.com/watch?v=O4L8f2GI1NM', 'https://www.youtube.com/watch?v=UH0v9pjo-I0', 'https://www.youtube.com/watch?v=efRsPcDuHmE', 'https://www.youtube.com/watch?v=g-HfLkiZsYg', 'https://www.youtube.com/watch?v=hCdGvWuF1d8', 'https://www.youtube.com/watch?v=Iaz6JIxDh6Y', 'https://www.youtube.com/watch?v=mn_MaruQIfo', 'https://www.youtube.com/watch?v=0bNIwlODieI', 'https://www.youtube.com/watch?v=jnZUHQHGTYM', 'https://www.youtube.com/watch?v=uiCeqySVhCE', 'https://www.youtube.com/watch?v=dd96_vUNsYQ', 'https://www.youtube.com/watch?v=YlUfufBj5vU', 'https://www.youtube.com/watch?v=X6mAf_uPy-o', 'https://www.youtube.com/watch?v=8LqzVjev4vg', 'https://www.youtube.com/watch?v=zRvqkUoiKJo', 'https://www.youtube.com/watch?v=URZihNBsnjA', 'https://www.youtube.com/watch?v=dg5ciDyauI4', 'https://www.youtube.com/watch?v=4plIX68mdE8', 'https://www.youtube.com/watch?v=ba_cJYpONvw', 'https://www.youtube.com/watch?v=SVk3Nh_seq0', 'https://www.youtube.com/watch?v=8ZWS0A5m8f8', 'https://www.youtube.com/watch?v=TZeCBVvH3bU', 'https://www.youtube.com/watch?v=WBSi93dzxPA', 'https://www.youtube.com/watch?v=OWhbSKlN3fE', 'https://www.youtube.com/watch?v=YvcCyRGOIYw', 'https://www.youtube.com/watch?v=3lcDT_-3v2k', 'https://www.youtube.com/watch?v=TYAjo3DCY6M', 'https://www.youtube.com/watch?v=ZiPXFeL3KQU', 'https://www.youtube.com/watch?v=cr69xk1xiOU', 'https://www.youtube.com/watch?v=omwcM_uesLs', 'https://www.youtube.com/watch?v=fkcGjy2wenk', 'https://www.youtube.com/watch?v=zTlSmQKqS-Q', 'https://www.youtube.com/watch?v=6YxPOGC2K8A', 'https://www.youtube.com/watch?v=3vpOmbGsLng', 'https://www.youtube.com/watch?v=E-yeVDKkRsM', 'https://www.youtube.com/watch?v=zIrIBP-BalQ', 'https://www.youtube.com/watch?v=d8Wrwo2a1dg', 'https://www.youtube.com/watch?v=WjIZEIDxFGc', 'https://www.youtube.com/watch?v=XhUOUb9mXek', 'https://www.youtube.com/watch?v=LNzQESt7g-0', 'https://www.youtube.com/watch?v=LmolEKym3rs', 'https://www.youtube.com/watch?v=xikoTE3zyME', 'https://www.youtube.com/watch?v=IEcJtk4-uZE', 'https://www.youtube.com/watch?v=apfgfG-Gmp4', 'https://www.youtube.com/watch?v=EA7Dj-dmzEE', 'https://www.youtube.com/watch?v=cMscNuSUy0I', 'https://www.youtube.com/watch?v=SL2IpMNTz1A', 'https://www.youtube.com/watch?v=mpIF7xjGyRs', 'https://www.youtube.com/watch?v=xebTB_Lh8I8', 'https://www.youtube.com/watch?v=WGdljrhdSKA', 'https://www.youtube.com/watch?v=9B_eLoVq4yI', 'https://www.youtube.com/watch?v=ea1p1eevcfo', 'https://www.youtube.com/watch?v=t-N3In2rLI4', 'https://www.youtube.com/watch?v=8wb6LFOYyQQ', 'https://www.youtube.com/watch?v=rQLq9iLQseA', 'https://www.youtube.com/watch?v=Vmh10WtC7ws', 'https://www.youtube.com/watch?v=A5MQw3N53MY', 'https://www.youtube.com/watch?v=d-6EXZ7yM2E', 'https://www.youtube.com/watch?v=39902cn5lX8', 'https://www.youtube.com/watch?v=lciwaaFQqCk', 'https://www.youtube.com/watch?v=w-wSasUe1hc', 'https://www.youtube.com/watch?v=zi6ae6kZNqE', 'https://www.youtube.com/watch?v=NaxkTCUtzX0', 'https://www.youtube.com/watch?v=widzhMeoAy0', 'https://www.youtube.com/watch?v=KZthDGH6WCU', 'https://www.youtube.com/watch?v=xOVuhgx0A0E', 'https://www.youtube.com/watch?v=nmguZTONFK4', 'https://www.youtube.com/watch?v=FUI71dZJ43k', 'https://www.youtube.com/watch?v=Fc0WAkPq_JU', 'https://www.youtube.com/watch?v=q0--w3_JlR4', 'https://www.youtube.com/watch?v=nV9vCx_vCVM', 'https://www.youtube.com/watch?v=DyEKsL-NSmw', 'https://www.youtube.com/watch?v=QsIBauyOdLk', 'https://www.youtube.com/watch?v=iQSdF5cs2YE', 'https://www.youtube.com/watch?v=lQ8JX_64E7Q', 'https://www.youtube.com/watch?v=dgt652020RY', 'https://www.youtube.com/watch?v=wUPZ8rSESZo', 'https://www.youtube.com/watch?v=HNZD0B1i1zM', 'https://www.youtube.com/watch?v=kP9pUqqeUD0', 'https://www.youtube.com/watch?v=OEdfJP9S4NE', 'https://www.youtube.com/watch?v=RYCS7LKu1UA', 'https://www.youtube.com/watch?v=yN5ESpYyXmY', 'https://www.youtube.com/watch?v=cdEZrvS_nnY', 'https://www.youtube.com/watch?v=ksUwluZXyWc', 'https://www.youtube.com/watch?v=TgbPQNjaSno', 'https://www.youtube.com/watch?v=1u2dwfRb9S8', 'https://www.youtube.com/watch?v=zZc5Z9C7UfU', 'https://www.youtube.com/watch?v=2xsaa6sN0cM', 'https://www.youtube.com/watch?v=wQJOljq3SZk', 'https://www.youtube.com/watch?v=Ro-kkxkHHuU', 'https://www.youtube.com/watch?v=XwQ-oZY4oKA', 'https://www.youtube.com/watch?v=Mk1U8MAudbk', 'https://www.youtube.com/watch?v=vq9irdLcZmU', 'https://www.youtube.com/watch?v=PHGuhEvdg6M', 'https://www.youtube.com/watch?v=jnc1Ay6X1bg', 'https://www.youtube.com/watch?v=hweZuG-UBmM', 'https://www.youtube.com/watch?v=EXBNGlUVgFs', 'https://www.youtube.com/watch?v=YhSUknaEnec', 'https://www.youtube.com/watch?v=W7QBt61ChR4', 'https://www.youtube.com/watch?v=kVNq8knHGew', 'https://www.youtube.com/watch?v=F5pJR45HNEM', 'https://www.youtube.com/watch?v=D5in5EdjhD0', 'https://www.youtube.com/watch?v=rV2KFnsfYPk', 'https://www.youtube.com/watch?v=DFaNajmJ2oc', 'https://www.youtube.com/watch?v=RMxKc-uFEnU', 'https://www.youtube.com/watch?v=1pH8gCEZ4Ds', 'https://www.youtube.com/watch?v=oUUkwmTmYqM', 'https://www.youtube.com/watch?v=CHS1NraVsAc', 'https://www.youtube.com/watch?v=U7EyfO0TRm4', 'https://www.youtube.com/watch?v=0XnihgXhh3c', 'https://www.youtube.com/watch?v=9DvmLMUfGss', 'https://www.youtube.com/watch?v=XNSxj0TVeJs', 'https://www.youtube.com/watch?v=-U56oPQado8', 'https://www.youtube.com/watch?v=2qCaW1_4LBQ', 'https://www.youtube.com/watch?v=uEGVwVaauDM', 'https://www.youtube.com/watch?v=2G6kf7XM9Nk', 'https://www.youtube.com/watch?v=2HvGy2gY0eM', 'https://www.youtube.com/watch?v=Grn2EZxcUeA', 'https://www.youtube.com/watch?v=-tU918oK85o', 'https://www.youtube.com/watch?v=Y3PwG4UoJ0Y', 'https://www.youtube.com/watch?v=h00PgJSL8wY', 'https://www.youtube.com/watch?v=8NTWv4qr3iA', 'https://www.youtube.com/watch?v=StGGngQIwY0', 'https://www.youtube.com/watch?v=L7ZK8J69CW0', 'https://www.youtube.com/watch?v=PHEKSSpOZGc', 'https://www.youtube.com/watch?v=2yVQOCzw34U', 'https://www.youtube.com/watch?v=5adE7kHqTLU', 'https://www.youtube.com/watch?v=guPlS0___ww', 'https://www.youtube.com/watch?v=Ah5RsGc5Tnk', 'https://www.youtube.com/watch?v=86RY40TvRgE', 'https://www.youtube.com/watch?v=-NGQ9INMnq0', 'https://www.youtube.com/watch?v=szIGZVrSAyc', 'https://www.youtube.com/watch?v=CvZRsdHgxgA', 'https://www.youtube.com/watch?v=mk8pxyAWTBk', 'https://www.youtube.com/watch?v=KYlyb1Bx9Ic', 'https://www.youtube.com/watch?v=8-jh2R-_eQY', 'https://www.youtube.com/watch?v=MXh1_ubCQAI', 'https://www.youtube.com/watch?v=Zh7ps1KWl_M', 'https://www.youtube.com/watch?v=uXPvPxOFba0', 'https://www.youtube.com/watch?v=-3kAld2YwgU', 'https://www.youtube.com/watch?v=nvo9O4A18d0', 'https://www.youtube.com/watch?v=5OA-KgruEEg', 'https://www.youtube.com/watch?v=fy69-u4rcHU', 'https://www.youtube.com/watch?v=4vGc0xdZsf4', 'https://www.youtube.com/watch?v=K_Gym829MIM', 'https://www.youtube.com/watch?v=GL1GcRrbPm0', 'https://www.youtube.com/watch?v=e8TihISaf7E', 'https://www.youtube.com/watch?v=9JScy7ulDpE', 'https://www.youtube.com/watch?v=TAP0xk-c4mk', 'https://www.youtube.com/watch?v=u0t34c3W8d0', 'https://www.youtube.com/watch?v=e1JWr03P9W8', 'https://www.youtube.com/watch?v=H-fRSrfr2L0', 'https://www.youtube.com/watch?v=tKFgMHFXaYs', 'https://www.youtube.com/watch?v=6gD-W-X6hdc', 'https://www.youtube.com/watch?v=WpJjuotD534', 'https://www.youtube.com/watch?v=VBI9mC77igo', 'https://www.youtube.com/watch?v=WSN_8YGX6W8', 'https://www.youtube.com/watch?v=QKq38COoTG8', 'https://www.youtube.com/watch?v=bgqwfCyZpao', 'https://www.youtube.com/watch?v=7TLZN92-dZo', 'https://www.youtube.com/watch?v=VoRmqbPFasE', 'https://www.youtube.com/watch?v=HDXmWCzw5_U', 'https://www.youtube.com/watch?v=1OGIJE8AzqM', 'https://www.youtube.com/watch?v=nv1eBf58JfI', 'https://www.youtube.com/watch?v=b-u9f5IJa1s', 'https://www.youtube.com/watch?v=lWGhJ63OXxM', 'https://www.youtube.com/watch?v=46bH_jV34ig', 'https://www.youtube.com/watch?v=hLsNOEfDrNw', 'https://www.youtube.com/watch?v=rH8SicnqSC4', 'https://www.youtube.com/watch?v=Oxr4KJtJgPc', #### 'https://www.youtube.com/watch?v=E-oqU5BPvI8', ] return urls_yt_add def urls_yt_remove(): urls_yt_remove = [ # The Forest Revealed 'https://www.youtube.com/watch?v=qAAX4olzVuQ', 'https://www.youtube.com/watch?v=KafxQqt-Ntw', 'https://www.youtube.com/watch?v=2s70bDIMs8c', 'https://www.youtube.com/watch?v=p4y8fTl00tI', 'https://www.youtube.com/watch?v=MKxXyt-YTss', 'https://www.youtube.com/watch?v=_mpZplkvm-s', 'https://www.youtube.com/watch?v=cOVl0ub0Mls', 'https://www.youtube.com/watch?v=kejMQ0tcdhs', 'https://www.youtube.com/watch?v=8x6uvAH0Yr8', 'https://www.youtube.com/watch?v=0M1G5DLnmfA', 'https://www.youtube.com/watch?v=0kpaVxnbWh4', 'https://www.youtube.com/watch?v=96Zt9l53xWk', 'https://www.youtube.com/watch?v=R6dZ_obn1NU', 'https://www.youtube.com/watch?v=38uAAYHGPYo', 'https://www.youtube.com/watch?v=yJzCNfdTf60', 'https://www.youtube.com/watch?v=7eNEfeIytMs', 'https://www.youtube.com/watch?v=OGQZtnyi4hc', 'https://www.youtube.com/watch?v=E6vLshKTOxQ', 'https://www.youtube.com/watch?v=-bLb7VZnqko', # Ricardo Ruiz 'https://www.youtube.com/watch?v=AUniGUu5hGE', 'https://www.youtube.com/watch?v=CooK_RWMreI', 'https://www.youtube.com/watch?v=05XGb2Aw5lA', 'https://www.youtube.com/watch?v=SDZYRvwI7pY', 'https://www.youtube.com/watch?v=VZYNRKKuaXY', # Chomskyan 'https://www.youtube.com/watch?v=HWgv0Pr1i9o', 'https://www.youtube.com/watch?v=7-iGi4H2Vqs', 'https://www.youtube.com/watch?v=UTnd8jG9Lfo', 'https://www.youtube.com/watch?v=eYr2wcsNGVc', 'https://www.youtube.com/watch?v=a27mnDbHCow', 'https://www.youtube.com/watch?v=4tfkiUtB3_0', 'https://www.youtube.com/watch?v=QNLHDsZZ50Q', 'https://www.youtube.com/watch?v=zIERifyW_aI', # Linguistics 'https://www.youtube.com/watch?v=ycC5oZqNXsk', 'https://www.youtube.com/watch?v=yRSxR9Us9QM', 'https://www.youtube.com/watch?v=msOnzHPeXzE', 'https://www.youtube.com/watch?v=24PcF7LDQ7M', # ...XCGIY 'https://www.youtube.com/watch?v=0lupJCD8DEw', # ...0xylW 'https://www.youtube.com/watch?v=O4GLzz-9lRY', 'https://www.youtube.com/watch?v=6-d23ma3qz8', 'https://www.youtube.com/watch?v=XUBYI97cUgU', 'https://www.youtube.com/watch?v=ww__v9i-PU4', 'https://www.youtube.com/watch?v=lWyuKzH9NSQ', 'https://www.youtube.com/watch?v=IyRoVOSVqZ8', # Ramin Zareian 'https://www.youtube.com/watch?v=U3sm-OUIDR4', # Unknown 'https://www.youtube.com/watch?v=wlrOyum4US0', 'https://www.youtube.com/watch?v=34LGPIXvU5M', 'https://www.youtube.com/watch?v=oDK0kJSvRm8', 'https://www.youtube.com/watch?v=44h9QuWcJYk', 'https://www.youtube.com/watch?v=zi6ae6kZNqE', 'https://www.youtube.com/watch?v=hwNrHEq2vI0' ] return urls_yt_remove # Remove duplicates from across YT channels/playlists/videos def urls_yt(): ydl = youtube_dl.YoutubeDL({'ignoreerrors': True, 'extract_flat': 'in_playlist', 'dump_single_json': True}) with ydl: yt_add_set = set() for url in urls_yt_add(): result = ydl.extract_info(url, download=False) if result is None: # Inaccessible video continue elif 'entries' in result: # Playlist yt_add_set.update(['https://www.youtube.com/watch?v=' + result_entry['url'] for result_entry in result['entries'] if result_entry is not None]) else: # Video yt_add_set.add(result['webpage_url']) return yt_add_set.difference(urls_yt_remove()) def chomsky_dot_info_articles(): # Get articles articles_url = "https://chomsky.info/articles/" req = Request(articles_url, headers={'User-Agent': 'Mozilla/5.0'}) page = urlopen(req) html = page.read().decode("utf-8") soup_articles = BeautifulSoup(html, "html.parser") urls = [a.get('href') for a in soup_articles.find_all('a')[23:-1]] urls = [url.replace('http://', 'https://') for url in urls] urls = ['https://chomsky.info/' + url[3:] if url.find('../') == 0 else url for url in urls] urls = ['https://chomsky.info/' + url[2:] if url.find('..') == 0 else url for url in urls] urls = [url.replace('zmag.org', 'zcomm.org') for url in urls] urls.remove('https://www.journals.uchicago.edu/doi/abs/10.1086/291699') urls.remove('https://www.chomsky.info/articles/200501--.pdf') urls.remove('https://www.chomsky.info/articles/200310--.pdf') urls.remove('https://www.chomsky.info/articles/20021122.pdf') urls.remove('https://www.chomsky.info/articles/2000----.pdf') urls.remove('https://www.chomsky.info/articles/1986----.pdf') return set(urls) def chomsky_dot_info_interviews(): # Get interviews interviews_url = "https://chomsky.info/interviews/" req = Request(interviews_url, headers={'User-Agent': 'Mozilla/5.0'}) page = urlopen(req) html = page.read().decode("utf-8") soup_articles = BeautifulSoup(html, "html.parser") urls = [a.get('href') for a in soup_articles.find_all('a')[23:-1]] urls = [url.replace('http://', 'https://') for url in urls] urls = ['https://chomsky.info/' + url[3:] if url.find('../') == 0 else url for url in urls] urls = ['https://chomsky.info/' + url[2:] if url.find('..') == 0 else url for url in urls] urls = [url.replace('zmag.org', 'zcomm.org') for url in urls] urls.remove('https://www.petemccormack.com/social_005.htm') urls.remove('https://smallbusiness.yahoo.com/geocities') urls.remove('https://www.nbcnews.com/id/3080909') return set(urls) def other(): 'https://theintercept.com/2019/10/31/deconstructed-special-the-noam-chomsky-interview/', 'https://couragefound.org/wp-content/uploads/2019/10/Letter-to-Permanent-Representatives.pdf', return def truthout_interviews(): 'https://truthout.org/articles/noam-chomsky-democratic-party-centrism-risks-handing-election-to-trump/', 'https://truthout.org/articles/chomsky-without-us-aid-israel-wouldnt-be-killing-palestinians-en-masse/', return