From 9dc7fd4c4cb07c0f82a91cafeafb3e1435af587f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 30 May 2022 22:47:47 +0800 Subject: [PATCH 01/10] Add test case for electrek --- .../electrek/expected-metadata.json | 12 + .../test/test-pages/electrek/expected.html | 78 + .../test/test-pages/electrek/source.html | 1887 +++++++++++++++++ .../test/test-pages/electrek/url.txt | 1 + 4 files changed, 1978 insertions(+) create mode 100644 packages/readabilityjs/test/test-pages/electrek/expected-metadata.json create mode 100644 packages/readabilityjs/test/test-pages/electrek/expected.html create mode 100644 packages/readabilityjs/test/test-pages/electrek/source.html create mode 100644 packages/readabilityjs/test/test-pages/electrek/url.txt diff --git a/packages/readabilityjs/test/test-pages/electrek/expected-metadata.json b/packages/readabilityjs/test/test-pages/electrek/expected-metadata.json new file mode 100644 index 000000000..ec043ebba --- /dev/null +++ b/packages/readabilityjs/test/test-pages/electrek/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "Georgia gives US solar panel manufacturing a big boost with a new factory", + "byline": "Michelle Lewis", + "dir": null, + "excerpt": "Solar-cell manufacturing giant Q Cells today announced that it's opening a new solar panel manufacturing facility in Dalton, Georgia.", + "siteName": "Electrek", + "siteIcon": "/favicon.ico", + "previewImage": "https://i0.wp.com/electrek.co/wp-content/uploads/sites/3/2022/05/georgia-solar-manufacturing.jpg?resize=1200%2C628&quality=82&strip=all&ssl=1", + "publishedDate": "2022-05-26T15:57:59.000Z", + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/electrek/expected.html b/packages/readabilityjs/test/test-pages/electrek/expected.html new file mode 100644 index 000000000..d1a72dd2a --- /dev/null +++ b/packages/readabilityjs/test/test-pages/electrek/expected.html @@ -0,0 +1,78 @@ +
+
+ +
+ +
+

Georgia solar manufacturing +

+ +
+
+

Seoul-headquartered PV solar-cell manufacturing giant Q Cells today announced that it’s opening a new solar panel manufacturing facility in Dalton, Georgia.

+

Georgia solar panel manufacturing grows again

+

It’s a $171 million expansion of Q Cells’ existing solar module manufacturing plant in Dalton, and that will create 470 additional jobs. Total local Q Cells employees will exceed 1,000 when the expansion is complete.

+

Groundbreaking is planned for fall 2022 and operation is expected to commence within the first half of 2023.

+

This latest domestic solar manufacturing expansion will boost production of advanced photovoltaic (PV) modules, and that will help the US work move toward its goal of decarbonizing the electric grid.

+

The new facility will produce 1.4 gigawatts (GW) of solar modules per year made with Q Cells’ next-gen PV cells, a high-efficiency tunnel oxide passivated contact technology better known as TOPCon.

+

Combined with the existing 1.7-GW factory, the expansion will bring Q Cells’ total capacity in the US to 3.1 GW; that’s equivalent to one-third of the country’s solar module manufacturing capacity.

+

Qcells CEO Justin Lee said:

+
+

Georgia has become the clean energy manufacturing heart of America, and we are proud to contribute to the state’s advanced manufacturing economy.

+
+

Q Cells has the largest market share in the US commercial and residential markets and also supplies the utility-scale solar sector.

+

Senator Jon Ossoff (D-GA) met with Q Cells’ parent company Hanwha in Seoul last year and has been actively pitching and securing additional clean energy investment in Georgia.

+
+
+

Ossoff also recently helped secure Hyundai’s investment in electric vehicles that will create over 8,000 jobs in Bryan County.

+

Washington, Georgia, embraces solar

+

Also in Georgia, WeSolar CSP, a minority-owned renewable energy tech and design company headquartered in Princeton, New Jersey, will design a solar farm along with a microgrid that will supply the City of Washington, Georgia, that will replace natural gas use. Washington is 90 miles east of Atlanta and has a population of around 4,000.

+

The project will comprise both solar panels and a concentrating solar-thermal power (CSP) technology.

+

WeSolar CSP’s CEO, Steve Anglin, said:

+
+

The citizens of the City of Washington will benefit by having a cleaner environment and experiencing price certainty in the face of the ever-increasing energy costs of fossil fuels.

+
+

+ Read more: Here are 3 vital insights installers shared about the state of solar in 2021 +

+

+ Photo: Hanwha Q Cells +

+
+

+ UnderstandSolar is a free service that links you to top-rated solar installers in your region for personalized solar estimates. Tesla now offers price matching, so it’s important to shop for the best quotes. Click here to learn more and get your quotes. — *ad. +

+

+ FTC: We use income earning auto affiliate links. More. +

+
+

+ Subscribe to Electrek on YouTube for exclusive videos and subscribe to the podcast. + +

+

+ +

+
+
+

About the Author

+
+

+

+ Michelle Lewis +

+

+ @michelle0728 +

+

Michelle Lewis is a writer and editor on Electrek and an editor on DroneDJ, 9to5Mac, and 9to5Google. She lives in White River Junction, Vermont. She has previously worked for Fast Company, the Guardian, News Deeply, Time, and others. Message Michelle on Twitter or at michelle@9to5mac.com. Check out her personal blog.

+
+
+

+ Michelle Lewis's favorite gear +

+
+
+
+ +
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/electrek/source.html b/packages/readabilityjs/test/test-pages/electrek/source.html new file mode 100644 index 000000000..b33526eb1 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/electrek/source.html @@ -0,0 +1,1887 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Georgia gives US solar panel manufacturing a big boost with a new factory + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+ +
+
+ +
+
+
+
+
+
+

+ May 26 +

+
+
+
+
+
+
+

+ Georgia gives US solar panel manufacturing a big boost with a new factory +

+
+

+ Michelle Lewis +

+

+ - May. 26th 2022 8:57 am PT +

+
+

+ +

+
+
+
+
+
+
+
+ Georgia solar manufacturing +
+
+ 35 Comments + +
+
+
+
+

+ Seoul-headquartered PV solar-cell manufacturing giant Q Cells today announced that it’s opening a new solar panel manufacturing facility in Dalton, Georgia. +

+

+ + +

+

+ Georgia solar panel manufacturing grows again +

+

+ It’s a $171 million expansion of Q Cells’ existing solar module manufacturing plant in Dalton, and that will create 470 additional jobs. Total local Q Cells employees will exceed 1,000 when the expansion is complete. +

+

+ Groundbreaking is planned for fall 2022 and operation is expected to commence within the first half of 2023. +

+

+ This latest domestic solar manufacturing expansion will boost production of advanced photovoltaic (PV) modules, and that will help the US work move toward its goal of decarbonizing the electric grid. +

+

+ The new facility will produce 1.4 gigawatts (GW) of solar modules per year made with Q Cells’ next-gen PV cells, a high-efficiency tunnel oxide passivated contact technology better known as TOPCon. +

+

+ Combined with the existing 1.7-GW factory, the expansion will bring Q Cells’ total capacity in the US to 3.1 GW; that’s equivalent to one-third of the country’s solar module manufacturing capacity. +

+

+ Qcells CEO Justin Lee said: +

+
+

+ Georgia has become the clean energy manufacturing heart of America, and we are proud to contribute to the state’s advanced manufacturing economy. +

+
+

+ Q Cells has the largest market share in the US commercial and residential markets and also supplies the utility-scale solar sector. +

+

+ Senator Jon Ossoff (D-GA) met with Q Cells’ parent company Hanwha in Seoul last year and has been actively pitching and securing additional clean energy investment in Georgia. +

+
+
+ +

+ +

+
+
+

+ Ossoff also recently helped secure Hyundai’s investment in electric vehicles that will create over 8,000 jobs in Bryan County. +

+

+ Washington, Georgia, embraces solar +

+

+ Also in Georgia, WeSolar CSP, a minority-owned renewable energy tech and design company headquartered in Princeton, New Jersey, will design a solar farm along with a microgrid that will supply the City of Washington, Georgia, that will replace natural gas use. Washington is 90 miles east of Atlanta and has a population of around 4,000. +

+

+ The project will comprise both solar panels and a concentrating solar-thermal power (CSP) technology. +

+

+ WeSolar CSP’s CEO, Steve Anglin, said: +

+
+

+ The citizens of the City of Washington will benefit by having a cleaner environment and experiencing price certainty in the face of the ever-increasing energy costs of fossil fuels. +

+
+

+ Read more: Here are 3 vital insights installers shared about the state of solar in 2021 +

+

+ Photo: Hanwha Q Cells +

+
+

+ UnderstandSolar is a free service that links you to top-rated solar installers in your region for personalized solar estimates. Tesla now offers price matching, so it’s important to shop for the best quotes. Click here to learn more and get your quotes. — *ad. +

+

+ +

+
+

+ FTC: We use income earning auto affiliate links. More. +

+
+
+ +
+
+
+ You’re reading Electrek— experts who break news about Tesla, electric vehicles, and green energy, day after day. Be sure to check out our homepage for all the latest news, and follow Electrek on Twitter, Facebook, and LinkedIn to stay in the loop. Don’t know where to start? Check out our YouTube channel for the latest reviews. +
+
+
+

+ Subscribe to Electrek on YouTube for exclusive videos and subscribe to the podcast. +

+
+ +
+
+
+
+
+

+ Guides +

+
+
+ +
+ Electrek Green Energy Brief +
+

+ Electrek Green Energy Brief +

+

+ Electrek Green Energy Brief: A daily technical, financial and political review/analysis of important green energy news +

+
+
+ +
+ Green Energy +
+

+ Green Energy +

+
+
+
+ EGEB Solar power Georgia +
+
+
+
+
+

+ About the Author +

+
+
+ +

+ Michelle Lewis +

+ +
+

+ Michelle Lewis is a writer and editor on Electrek and an editor on DroneDJ, 9to5Mac, and 9to5Google. She lives in White River Junction, Vermont. She has previously worked for Fast Company, the Guardian, News Deeply, Time, and others. Message Michelle on Twitter or at michelle@9to5mac.com. Check out her personal blog. +

+
+
+
+
+

+ Michelle Lewis's favorite gear +

+
+
+ +
+ NordVPN +
+

+ NordVPN +

+
+ Because I don't want to wait for the best of British TV. +
+
+
+ +
+ MacBook Air +
+

+ MacBook Air +

+
+ Light, durable, quick: I'll never go back. +
+
+
+
+
+
+
+
+
+ +
+
+
+
+ +
+
+
+ +
+
+
+
+
+ Powered by WordPress VIP +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/readabilityjs/test/test-pages/electrek/url.txt b/packages/readabilityjs/test/test-pages/electrek/url.txt new file mode 100644 index 000000000..04b170cf7 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/electrek/url.txt @@ -0,0 +1 @@ +https://electrek.co/2022/05/26/georgia-solar-panel-manufacturing/ \ No newline at end of file From bce50c2a92ddb7a4cb85eb3953c7b840e33d64cf Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 31 May 2022 11:11:06 +0800 Subject: [PATCH 02/10] Update electrek test case --- .../test/test-pages/electrek/expected.html | 53 +- .../test/test-pages/electrek/source.html | 6602 ++++++++++++----- 2 files changed, 4786 insertions(+), 1869 deletions(-) diff --git a/packages/readabilityjs/test/test-pages/electrek/expected.html b/packages/readabilityjs/test/test-pages/electrek/expected.html index d1a72dd2a..29df338f2 100644 --- a/packages/readabilityjs/test/test-pages/electrek/expected.html +++ b/packages/readabilityjs/test/test-pages/electrek/expected.html @@ -4,10 +4,19 @@
-

Georgia solar manufacturing -

+

Georgia solar manufacturing +

+ -
+ +

Seoul-headquartered PV solar-cell manufacturing giant Q Cells today announced that it’s opening a new solar panel manufacturing facility in Dalton, Georgia.

Georgia solar panel manufacturing grows again

@@ -16,7 +25,7 @@

This latest domestic solar manufacturing expansion will boost production of advanced photovoltaic (PV) modules, and that will help the US work move toward its goal of decarbonizing the electric grid.

The new facility will produce 1.4 gigawatts (GW) of solar modules per year made with Q Cells’ next-gen PV cells, a high-efficiency tunnel oxide passivated contact technology better known as TOPCon.

Combined with the existing 1.7-GW factory, the expansion will bring Q Cells’ total capacity in the US to 3.1 GW; that’s equivalent to one-third of the country’s solar module manufacturing capacity.

-

Qcells CEO Justin Lee said:

+

Qcells CEO Justin Lee said:

Georgia has become the clean energy manufacturing heart of America, and we are proud to contribute to the state’s advanced manufacturing economy.

@@ -28,22 +37,22 @@

Washington, Georgia, embraces solar

Also in Georgia, WeSolar CSP, a minority-owned renewable energy tech and design company headquartered in Princeton, New Jersey, will design a solar farm along with a microgrid that will supply the City of Washington, Georgia, that will replace natural gas use. Washington is 90 miles east of Atlanta and has a population of around 4,000.

The project will comprise both solar panels and a concentrating solar-thermal power (CSP) technology.

-

WeSolar CSP’s CEO, Steve Anglin, said:

+

WeSolar CSP’s CEO, Steve Anglin, said:

The citizens of the City of Washington will benefit by having a cleaner environment and experiencing price certainty in the face of the ever-increasing energy costs of fossil fuels.

- Read more: Here are 3 vital insights installers shared about the state of solar in 2021 -

-

- Photo: Hanwha Q Cells + Read more: + Here are 3 vital insights installers shared about the state of solar in 2021

+

Photo: Hanwha Q Cells

UnderstandSolar is a free service that links you to top-rated solar installers in your region for personalized solar estimates. Tesla now offers price matching, so it’s important to shop for the best quotes. Click here to learn more and get your quotes. — *ad.

- FTC: We use income earning auto affiliate links. More. + FTC: We use income earning auto affiliate links. + More.

@@ -51,28 +60,32 @@

- +

-
+ +
-

About the Author

+

About the Author

-

+

+ +

- Michelle Lewis + + Michelle Lewis +

- @michelle0728 + @michelle0728

Michelle Lewis is a writer and editor on Electrek and an editor on DroneDJ, 9to5Mac, and 9to5Google. She lives in White River Junction, Vermont. She has previously worked for Fast Company, the Guardian, News Deeply, Time, and others. Message Michelle on Twitter or at michelle@9to5mac.com. Check out her personal blog.

-

- Michelle Lewis's favorite gear -

+

Michelle Lewis's favorite gear

-
+ + \ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/electrek/source.html b/packages/readabilityjs/test/test-pages/electrek/source.html index b33526eb1..7e4fb52c1 100644 --- a/packages/readabilityjs/test/test-pages/electrek/source.html +++ b/packages/readabilityjs/test/test-pages/electrek/source.html @@ -1,98 +1,489 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - Georgia gives US solar panel manufacturing a big boost with a new factory - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - fieldset.bbp-form { + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ +
+
+
+ +
+ +
+ +
+
+ + +
+
+
+
+

May 26

+
+ +
+ +
+ +
+ +
+
+

+ Georgia gives US solar panel manufacturing a big boost with a new + factory +

+
+

+ + Michelle Lewis + +

+

- May. 26th 2022 8:57 am PT

+
+

+ +

+

+ +
+
+ +
+ +
+ +
+ +
+
+ Georgia solar manufacturing +
+ +
+ 35 Comments + +
+ +
+ +
+
+

+ Seoul-headquartered PV solar-cell manufacturing giant + Q Cells + today announced that it’s opening a new solar panel manufacturing + facility in Dalton, Georgia. +

+

+ + +

+

+ Georgia solar panel manufacturing grows again +

+

+ It’s a $171 million expansion of Q Cells’ existing solar module + manufacturing plant in Dalton, and that will create 470 additional + jobs. Total local Q Cells employees will exceed 1,000 when the + expansion is complete. +

+

+ Groundbreaking is planned for fall 2022 and operation is expected + to commence within the first half of 2023. +

+

+ This latest domestic solar manufacturing expansion will boost + production of advanced photovoltaic (PV) modules, and that will + help the US work move toward its goal of decarbonizing the + electric grid. +

+

+ The new facility will produce 1.4 gigawatts (GW) of solar modules + per year made with Q Cells’ next-gen PV cells, a high-efficiency + tunnel oxide passivated contact technology better known as TOPCon. +

+

+ Combined with the existing 1.7-GW factory, the expansion will + bring Q Cells’ total capacity in the US to 3.1 GW; that’s + equivalent to one-third of the country’s solar module + manufacturing capacity. +

+

Qcells CEO Justin Lee said:

+
+

+ Georgia has become the clean energy manufacturing heart of + America, and we are proud to contribute to the state’s advanced + manufacturing economy. +

+
+

+ Q Cells has the largest market share in the US commercial and + residential markets and also supplies the utility-scale solar + sector. +

+

+ Senator Jon Ossoff (D-GA) met with Q Cells’ parent company + Hanwha in Seoul last year and has been actively pitching and + securing additional clean energy investment in Georgia. +

+
+
+ + +

+ +

+
+
+

+ Ossoff also recently helped secure Hyundai’s investment in electric vehicles + that will create over 8,000 jobs in Bryan County. +

+

+ Washington, Georgia, embraces solar +

+

+ Also in Georgia, + WeSolar CSP, a minority-owned renewable energy tech and design company + headquartered in Princeton, New Jersey, will design a solar farm + along with a microgrid that will supply the City of Washington, + Georgia, that will replace natural gas use. Washington is 90 miles + east of Atlanta and has a population of around 4,000. +

+

+ The project will comprise both solar panels and a concentrating + solar-thermal power (CSP) technology. +

+

WeSolar CSP’s CEO, Steve Anglin, said:

+
+

+ The citizens of the City of Washington will benefit by having a + cleaner environment and experiencing price certainty in the face + of the ever-increasing energy costs of fossil fuels. +

+
+

+ Read more: + Here are 3 vital insights installers shared about the state of + solar in 2021 +

+

Photo: Hanwha Q Cells

+
+

+ UnderstandSolar is a free service that links you to top-rated + solar installers in your region for personalized solar + estimates. Tesla now offers price matching, so it’s important to + shop for the best quotes. Click here to learn more and get your quotes. — *ad. +

+

+ +

+
+

+ FTC: We use income earning auto affiliate links. + More. +

+ +
+
+ +
+
+
+ You’re reading Electrek— experts who break news about + Tesla, + electric vehicles, + and green energy, + day after day. Be sure to check out our + homepage for all the latest + news, and follow Electrek on + Twitter, + Facebook, and + LinkedIn + to stay in the loop. Don’t know where to start? Check out our + YouTube channel for the latest reviews. +
+
+
+

+ Subscribe to Electrek on YouTube for exclusive videos + and subscribe to the + podcast. +

+
+ +
+
+ +
+ + +
+
+

Guides

+
+
+ +
+ Electrek Green Energy Brief +
+

Electrek Green Energy Brief

+

+ Electrek Green Energy Brief: A daily technical, financial + and political review/analysis of important green energy news +

+ +
+
+ +
+ Green Energy +
+

Green Energy

+

+ +
+
+
+ EGEB + Solar power + Georgia +
+ +
+
+ +
+
+

About the Author

+
+
+ + + +

+ + Michelle Lewis + +

+ + + +
+

+ Michelle Lewis is a writer and editor on Electrek and an + editor on DroneDJ, 9to5Mac, and 9to5Google. She lives in + White River Junction, Vermont. She has previously worked for + Fast Company, the Guardian, News Deeply, Time, and others. + Message Michelle on Twitter or at michelle@9to5mac.com. + Check out her personal blog. +

+
+
+
+
+

Michelle Lewis's favorite gear

+
+
+ +
+ NordVPN +
+

NordVPN

+
+ Because I don't want to wait for the best of British TV. +
+ +
+
+ +
+ MacBook Air +
+

MacBook Air

+
+ Light, durable, quick: I'll never go back. +
+ +
+
+
+
+
+ +
+
+
+
+
+
+ + + +
+
+
+ +
+
+ +
+ +
+ + +
+ Powered by WordPress VIP +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - -
-
-
-
- -
- -
-
- -
-
-
-
-
-
-

- May 26 -

-
-
-
-
-
-
-

- Georgia gives US solar panel manufacturing a big boost with a new factory -

-
-

- Michelle Lewis -

-

- - May. 26th 2022 8:57 am PT -

-
-

- -

-
-
-
-
-
-
-
- Georgia solar manufacturing -
-
- 35 Comments - -
-
-
-
-

- Seoul-headquartered PV solar-cell manufacturing giant Q Cells today announced that it’s opening a new solar panel manufacturing facility in Dalton, Georgia. -

-

- - -

-

- Georgia solar panel manufacturing grows again -

-

- It’s a $171 million expansion of Q Cells’ existing solar module manufacturing plant in Dalton, and that will create 470 additional jobs. Total local Q Cells employees will exceed 1,000 when the expansion is complete. -

-

- Groundbreaking is planned for fall 2022 and operation is expected to commence within the first half of 2023. -

-

- This latest domestic solar manufacturing expansion will boost production of advanced photovoltaic (PV) modules, and that will help the US work move toward its goal of decarbonizing the electric grid. -

-

- The new facility will produce 1.4 gigawatts (GW) of solar modules per year made with Q Cells’ next-gen PV cells, a high-efficiency tunnel oxide passivated contact technology better known as TOPCon. -

-

- Combined with the existing 1.7-GW factory, the expansion will bring Q Cells’ total capacity in the US to 3.1 GW; that’s equivalent to one-third of the country’s solar module manufacturing capacity. -

-

- Qcells CEO Justin Lee said: -

-
-

- Georgia has become the clean energy manufacturing heart of America, and we are proud to contribute to the state’s advanced manufacturing economy. -

-
-

- Q Cells has the largest market share in the US commercial and residential markets and also supplies the utility-scale solar sector. -

-

- Senator Jon Ossoff (D-GA) met with Q Cells’ parent company Hanwha in Seoul last year and has been actively pitching and securing additional clean energy investment in Georgia. -

-
-
- -

- -

-
-
-

- Ossoff also recently helped secure Hyundai’s investment in electric vehicles that will create over 8,000 jobs in Bryan County. -

-

- Washington, Georgia, embraces solar -

-

- Also in Georgia, WeSolar CSP, a minority-owned renewable energy tech and design company headquartered in Princeton, New Jersey, will design a solar farm along with a microgrid that will supply the City of Washington, Georgia, that will replace natural gas use. Washington is 90 miles east of Atlanta and has a population of around 4,000. -

-

- The project will comprise both solar panels and a concentrating solar-thermal power (CSP) technology. -

-

- WeSolar CSP’s CEO, Steve Anglin, said: -

-
-

- The citizens of the City of Washington will benefit by having a cleaner environment and experiencing price certainty in the face of the ever-increasing energy costs of fossil fuels. -

-
-

- Read more: Here are 3 vital insights installers shared about the state of solar in 2021 -

-

- Photo: Hanwha Q Cells -

-
-

- UnderstandSolar is a free service that links you to top-rated solar installers in your region for personalized solar estimates. Tesla now offers price matching, so it’s important to shop for the best quotes. Click here to learn more and get your quotes. — *ad. -

-

- -

-
-

- FTC: We use income earning auto affiliate links. More. -

-
-
- -
-
-
- You’re reading Electrek— experts who break news about Tesla, electric vehicles, and green energy, day after day. Be sure to check out our homepage for all the latest news, and follow Electrek on Twitter, Facebook, and LinkedIn to stay in the loop. Don’t know where to start? Check out our YouTube channel for the latest reviews. -
-
-
-

- Subscribe to Electrek on YouTube for exclusive videos and subscribe to the podcast. -

-
- -
-
-
-
-
-

- Guides -

-
-
- -
- Electrek Green Energy Brief -
-

- Electrek Green Energy Brief -

-

- Electrek Green Energy Brief: A daily technical, financial and political review/analysis of important green energy news -

-
-
- -
- Green Energy -
-

- Green Energy -

-
-
-
- EGEB Solar power Georgia -
-
-
-
-
-

- About the Author -

-
-
- -

- Michelle Lewis -

- -
-

- Michelle Lewis is a writer and editor on Electrek and an editor on DroneDJ, 9to5Mac, and 9to5Google. She lives in White River Junction, Vermont. She has previously worked for Fast Company, the Guardian, News Deeply, Time, and others. Message Michelle on Twitter or at michelle@9to5mac.com. Check out her personal blog. -

-
-
-
-
-

- Michelle Lewis's favorite gear -

-
-
- -
- NordVPN -
-

- NordVPN -

-
- Because I don't want to wait for the best of British TV. -
-
-
- -
- MacBook Air -
-

- MacBook Air -

-
- Light, durable, quick: I'll never go back. -
-
-
-
-
-
-
-
-
- -
-
-
-
- -
-
-
- -
-
-
-
-
- Powered by WordPress VIP -
- - - - - - - - - - - - - - - - - - - - - - - - - - + width: 0px; + height: 0px; + border: none; + z-index: -1000; + left: -1000px; + top: -1000px; + " + > + + From cc5bdf96f563d58f34e87b42a8cd080d205f12fb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 31 May 2022 12:52:33 +0800 Subject: [PATCH 03/10] Update electrek test case --- .../test/test-pages/electrek/expected.html | 2 +- .../test/test-pages/electrek/source.html | 197 +++++++++++++----- 2 files changed, 144 insertions(+), 55 deletions(-) diff --git a/packages/readabilityjs/test/test-pages/electrek/expected.html b/packages/readabilityjs/test/test-pages/electrek/expected.html index 29df338f2..c20fa5f5d 100644 --- a/packages/readabilityjs/test/test-pages/electrek/expected.html +++ b/packages/readabilityjs/test/test-pages/electrek/expected.html @@ -60,7 +60,7 @@

- +

diff --git a/packages/readabilityjs/test/test-pages/electrek/source.html b/packages/readabilityjs/test/test-pages/electrek/source.html index 7e4fb52c1..fa85637fd 100644 --- a/packages/readabilityjs/test/test-pages/electrek/source.html +++ b/packages/readabilityjs/test/test-pages/electrek/source.html @@ -9,20 +9,19 @@ name="robots" content="index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1" /> - + - + - + + + +
@@ -2030,7 +2037,7 @@
-
-
-
+
+

Seoul-headquartered PV solar-cell manufacturing giant -

+

@@ -2326,14 +2333,14 @@ flex-grow: 1; " title="Twitter Tweet" - src="https://platform.twitter.com/embed/Tweet.html?creatorScreenName=michelle0728&dnt=true&embedId=twitter-widget-0&features=eyJ0ZndfZXhwZXJpbWVudHNfY29va2llX2V4cGlyYXRpb24iOnsiYnVja2V0IjoxMjA5NjAwLCJ2ZXJzaW9uIjpudWxsfSwidGZ3X3NlbnNpdGl2ZV9tZWRpYV9pbnRlcnN0aXRpYWxfMTM5NjMiOnsiYnVja2V0IjoiaW50ZXJzdGl0aWFsIiwidmVyc2lvbiI6bnVsbH0sInRmd190d2VldF9yZXN1bHRfbWlncmF0aW9uXzEzOTc5Ijp7ImJ1Y2tldCI6InR3ZWV0X3Jlc3VsdCIsInZlcnNpb24iOm51bGx9fQ%3D%3D&frame=false&hideCard=false&hideThread=false&id=1529811248258371590&lang=en&origin=https%3A%2F%2Felectrek.co%2F2022%2F05%2F26%2Fgeorgia-solar-panel-manufacturing%2F&sessionId=287611a9e27150539e4b1dbe25290783e0e648ef&siteScreenName=electrekco&theme=light&widgetsVersion=c8fe9736dd6fb%3A1649830956492&width=550px" + src="https://platform.twitter.com/embed/Tweet.html?creatorScreenName=michelle0728&dnt=true&embedId=twitter-widget-0&features=eyJ0ZndfZXhwZXJpbWVudHNfY29va2llX2V4cGlyYXRpb24iOnsiYnVja2V0IjoxMjA5NjAwLCJ2ZXJzaW9uIjpudWxsfSwidGZ3X3NlbnNpdGl2ZV9tZWRpYV9pbnRlcnN0aXRpYWxfMTM5NjMiOnsiYnVja2V0IjoiaW50ZXJzdGl0aWFsIiwidmVyc2lvbiI6bnVsbH0sInRmd190d2VldF9yZXN1bHRfbWlncmF0aW9uXzEzOTc5Ijp7ImJ1Y2tldCI6InR3ZWV0X3Jlc3VsdCIsInZlcnNpb24iOm51bGx9fQ%3D%3D&frame=false&hideCard=false&hideThread=false&id=1529811248258371590&lang=en&origin=https%3A%2F%2Felectrek.co%2F2022%2F05%2F26%2Fgeorgia-solar-panel-manufacturing%2F&sessionId=3ecedb147f8f282e12ae20fb3f699df5891344fb&siteScreenName=electrekco&theme=light&widgetsVersion=c8fe9736dd6fb%3A1649830956492&width=550px" data-tweet-id="1529811248258371590" >

@@ -4087,8 +4092,18 @@ src="https://platform.twitter.com/widgets/widget_iframe.bbd13993eb53d3a11ac08f5e8cf9d6a4.html?origin=https%3A%2F%2Felectrek.co" title="Twitter settings iframe" style="display: none" - > + + + - - - +
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+ You are seeing this message because ad or script blocking software + is interfering with this page. +
+
+
+
+
+
+ Disable any ad or script blocking software, then reload this page. +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-

+
+
+

Seoul-headquartered PV solar-cell manufacturing giant Q Cells today announced that it’s opening a new solar panel manufacturing facility in Dalton, Georgia.

+

Georgia solar panel manufacturing grows again

+

It’s a $171 million expansion of Q Cells’ existing solar module manufacturing plant in Dalton, and that will create 470 additional jobs. Total local Q Cells employees will exceed 1,000 when the expansion is complete.

+

Groundbreaking is planned for fall 2022 and operation is expected to commence within the first half of 2023.

+

This latest domestic solar manufacturing expansion will boost production of advanced photovoltaic (PV) modules, and that will help the US work move toward its goal of decarbonizing the electric grid.

+

The new facility will produce 1.4 gigawatts (GW) of solar modules per year made with Q Cells’ next-gen PV cells, a high-efficiency tunnel oxide passivated contact technology better known as TOPCon.

+

Combined with the existing 1.7-GW factory, the expansion will bring Q Cells’ total capacity in the US to 3.1 GW; that’s equivalent to one-third of the country’s solar module manufacturing capacity.

+

Qcells CEO Justin Lee said:

+
+

Georgia has become the clean energy manufacturing heart of America, and we are proud to contribute to the state’s advanced manufacturing economy.

+
+

Q Cells has the largest market share in the US commercial and residential markets and also supplies the utility-scale solar sector.

+

Senator Jon Ossoff (D-GA) met with Q Cells’ parent company Hanwha in Seoul last year and has been actively pitching and securing additional clean energy investment in Georgia.

+
+
+
+
+
+

Ossoff also recently helped secure Hyundai’s investment in electric vehicles that will create over 8,000 jobs in Bryan County.

+

Washington, Georgia, embraces solar

+

Also in Georgia, WeSolar CSP, a minority-owned renewable energy tech and design company headquartered in Princeton, New Jersey, will design a solar farm along with a microgrid that will supply the City of Washington, Georgia, that will replace natural gas use. Washington is 90 miles east of Atlanta and has a population of around 4,000.

+

The project will comprise both solar panels and a concentrating solar-thermal power (CSP) technology.

+

WeSolar CSP’s CEO, Steve Anglin, said:

+
+

The citizens of the City of Washington will benefit by having a cleaner environment and experiencing price certainty in the face of the ever-increasing energy costs of fossil fuels.

+
+

+ Read more: + Here are 3 vital insights installers shared about the state of solar in 2021 +

+

Photo: Hanwha Q Cells

+
+

+ UnderstandSolar is a free service that links you to top-rated solar installers in your region for personalized solar estimates. Tesla now offers price matching, so it’s important to shop for the best quotes. Click here to learn more and get your quotes. — *ad. +

+

+ FTC: We use income earning auto affiliate links. + More. +

+
+

+ Subscribe to Electrek on YouTube for exclusive videos and subscribe to the podcast. + +

+

+ +

+
+
-
-

About the Author

-
+
+

About the Author

+

- +

Michelle Lewis

-

- @michelle0728 +

Michelle Lewis is a writer and editor on Electrek and an editor on DroneDJ, 9to5Mac, and 9to5Google. She lives in White River Junction, Vermont. She has previously worked for Fast Company, the Guardian, News Deeply, Time, and others. Message Michelle on Twitter or at michelle@9to5mac.com. Check out her personal blog.

-
+

Michelle Lewis's favorite gear

From b6fef171bec6da412431497b401669a15cdfceff Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 31 May 2022 14:18:39 +0800 Subject: [PATCH 05/10] If we have a node with only one child element which has the placeholder class, keep it --- packages/readabilityjs/Readability.js | 8 +- .../test/test-pages/electrek/expected.html | 117 +++++++++--------- 2 files changed, 64 insertions(+), 61 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index d3559b160..33e66d52d 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -232,7 +232,7 @@ Readability.prototype = { PLACEHOLDER_CLASSES: ['tweet-placeholder', 'instagram-placeholder'], // Classes of embeds extracted by the extension - EMBEDS_CLASSES: ['omnivore-instagram-embed', 'tweet-placeholder'], + EMBEDS_CLASSES: ['omnivore-instagram-embed'], // These are the list of HTML entities that need to be escaped. HTML_ESCAPE_MAP: { @@ -591,6 +591,12 @@ Readability.prototype = { continue; } + // If we have a node with only one child element which has the placeholder class, keep it + if (this._hasSingleTagInsideElement(node, "DIV") && this.PLACEHOLDER_CLASSES.includes(node.firstElementChild.className)) { + node = this._getNextNode(node); + continue; + } + if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) { if (this._isElementWithoutContent(node)) { node = this._removeAndGetNext(node); diff --git a/packages/readabilityjs/test/test-pages/electrek/expected.html b/packages/readabilityjs/test/test-pages/electrek/expected.html index 81483865f..927d09d32 100644 --- a/packages/readabilityjs/test/test-pages/electrek/expected.html +++ b/packages/readabilityjs/test/test-pages/electrek/expected.html @@ -1,9 +1,9 @@
-
+
-
+

-

-

Seoul-headquartered PV solar-cell manufacturing giant Q Cells today announced that it’s opening a new solar panel manufacturing facility in Dalton, Georgia.

-

Georgia solar panel manufacturing grows again

-

It’s a $171 million expansion of Q Cells’ existing solar module manufacturing plant in Dalton, and that will create 470 additional jobs. Total local Q Cells employees will exceed 1,000 when the expansion is complete.

-

Groundbreaking is planned for fall 2022 and operation is expected to commence within the first half of 2023.

-

This latest domestic solar manufacturing expansion will boost production of advanced photovoltaic (PV) modules, and that will help the US work move toward its goal of decarbonizing the electric grid.

-

The new facility will produce 1.4 gigawatts (GW) of solar modules per year made with Q Cells’ next-gen PV cells, a high-efficiency tunnel oxide passivated contact technology better known as TOPCon.

-

Combined with the existing 1.7-GW factory, the expansion will bring Q Cells’ total capacity in the US to 3.1 GW; that’s equivalent to one-third of the country’s solar module manufacturing capacity.

-

Qcells CEO Justin Lee said:

-
-

Georgia has become the clean energy manufacturing heart of America, and we are proud to contribute to the state’s advanced manufacturing economy.

-
-

Q Cells has the largest market share in the US commercial and residential markets and also supplies the utility-scale solar sector.

-

Senator Jon Ossoff (D-GA) met with Q Cells’ parent company Hanwha in Seoul last year and has been actively pitching and securing additional clean energy investment in Georgia.

-
-
-
-
-
-

Ossoff also recently helped secure Hyundai’s investment in electric vehicles that will create over 8,000 jobs in Bryan County.

-

Washington, Georgia, embraces solar

-

Also in Georgia, WeSolar CSP, a minority-owned renewable energy tech and design company headquartered in Princeton, New Jersey, will design a solar farm along with a microgrid that will supply the City of Washington, Georgia, that will replace natural gas use. Washington is 90 miles east of Atlanta and has a population of around 4,000.

-

The project will comprise both solar panels and a concentrating solar-thermal power (CSP) technology.

-

WeSolar CSP’s CEO, Steve Anglin, said:

-
-

The citizens of the City of Washington will benefit by having a cleaner environment and experiencing price certainty in the face of the ever-increasing energy costs of fossil fuels.

-
-

- Read more: - Here are 3 vital insights installers shared about the state of solar in 2021 -

-

Photo: Hanwha Q Cells

-
-

- UnderstandSolar is a free service that links you to top-rated solar installers in your region for personalized solar estimates. Tesla now offers price matching, so it’s important to shop for the best quotes. Click here to learn more and get your quotes. — *ad. -

-

- FTC: We use income earning auto affiliate links. - More. -

-
-

- Subscribe to Electrek on YouTube for exclusive videos and subscribe to the podcast. - -

-

- -

-
- +
+

Seoul-headquartered PV solar-cell manufacturing giant Q Cells today announced that it’s opening a new solar panel manufacturing facility in Dalton, Georgia.

+

Georgia solar panel manufacturing grows again

+

It’s a $171 million expansion of Q Cells’ existing solar module manufacturing plant in Dalton, and that will create 470 additional jobs. Total local Q Cells employees will exceed 1,000 when the expansion is complete.

+

Groundbreaking is planned for fall 2022 and operation is expected to commence within the first half of 2023.

+

This latest domestic solar manufacturing expansion will boost production of advanced photovoltaic (PV) modules, and that will help the US work move toward its goal of decarbonizing the electric grid.

+

The new facility will produce 1.4 gigawatts (GW) of solar modules per year made with Q Cells’ next-gen PV cells, a high-efficiency tunnel oxide passivated contact technology better known as TOPCon.

+

Combined with the existing 1.7-GW factory, the expansion will bring Q Cells’ total capacity in the US to 3.1 GW; that’s equivalent to one-third of the country’s solar module manufacturing capacity.

+

Qcells CEO Justin Lee said:

+
+

Georgia has become the clean energy manufacturing heart of America, and we are proud to contribute to the state’s advanced manufacturing economy.

+
+

Q Cells has the largest market share in the US commercial and residential markets and also supplies the utility-scale solar sector.

+

Senator Jon Ossoff (D-GA) met with Q Cells’ parent company Hanwha in Seoul last year and has been actively pitching and securing additional clean energy investment in Georgia.

+
+
+
+
+
+

Ossoff also recently helped secure Hyundai’s investment in electric vehicles that will create over 8,000 jobs in Bryan County.

+

Washington, Georgia, embraces solar

+

Also in Georgia, WeSolar CSP, a minority-owned renewable energy tech and design company headquartered in Princeton, New Jersey, will design a solar farm along with a microgrid that will supply the City of Washington, Georgia, that will replace natural gas use. Washington is 90 miles east of Atlanta and has a population of around 4,000.

+

The project will comprise both solar panels and a concentrating solar-thermal power (CSP) technology.

+

WeSolar CSP’s CEO, Steve Anglin, said:

+
+

The citizens of the City of Washington will benefit by having a cleaner environment and experiencing price certainty in the face of the ever-increasing energy costs of fossil fuels.

+
+

+ Read more: + Here are 3 vital insights installers shared about the state of solar in 2021 +

+

Photo: Hanwha Q Cells

+
+

+ UnderstandSolar is a free service that links you to top-rated solar installers in your region for personalized solar estimates. Tesla now offers price matching, so it’s important to shop for the best quotes. Click here to learn more and get your quotes. — *ad. +

+

+ FTC: We use income earning auto affiliate links. + More. +

+
+

+ Subscribe to Electrek on YouTube for exclusive videos and subscribe to the podcast. + +

+

+ +

-
-

About the Author

-
+
+

About the Author

+

- +

Michelle Lewis

-

+ @michelle0728

Michelle Lewis is a writer and editor on Electrek and an editor on DroneDJ, 9to5Mac, and 9to5Google. She lives in White River Junction, Vermont. She has previously worked for Fast Company, the Guardian, News Deeply, Time, and others. Message Michelle on Twitter or at michelle@9to5mac.com. Check out her personal blog.

-
+

Michelle Lewis's favorite gear

From 417ed0a4eb7769b5dc10863255161735e47778cc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 31 May 2022 20:02:54 +0800 Subject: [PATCH 06/10] Fetch tweet id from url --- packages/readabilityjs/Readability.js | 27 +- packages/readabilityjs/package.json | 3 +- .../expected-metadata.json | 10 + .../expected.html | 38 + .../substack-michaelshellenberger/source.html | 2000 +++++++++++++++++ .../substack-michaelshellenberger/url.txt | 1 + 6 files changed, 2073 insertions(+), 6 deletions(-) create mode 100644 packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json create mode 100644 packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html create mode 100644 packages/readabilityjs/test/test-pages/substack-michaelshellenberger/source.html create mode 100644 packages/readabilityjs/test/test-pages/substack-michaelshellenberger/url.txt diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 33e66d52d..f322a2113 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -22,6 +22,7 @@ var parseSrcset = require('parse-srcset'); var htmlEntities = require('html-entities') +const axios = require("axios"); /** Checks whether an element is a wrapper for tweet */ const hasTweetInChildren = element => { @@ -2204,15 +2205,15 @@ Readability.prototype = { } }, - _createPlaceholders: function (e) { - Array.from(e.getElementsByTagName('a')).forEach(element => { + _createPlaceholders: async function (e) { + for (const element of Array.from(e.getElementsByTagName('a'))) { if (this.isEmbed(element)) { - return; + continue; } // Create tweets placeholders from links - if (element.href.includes('twitter.com')) { + if (element.href.includes('twitter.com') || element.parentNode.className === 'tweet') { const link = element.href; const regex = /(https?:\/\/twitter\.com\/\w+\/status\/)(\d+)/gm; const match = regex.exec(link); @@ -2234,6 +2235,22 @@ Readability.prototype = { if (tweetParent && tweetParent.className.includes('twitter-tweet')) { tweetParent.parentNode.replaceChild(tweet, tweetParent); } + } else if (element.parentNode.className === 'tweet') { + // Create tweets placeholders from classname + try { + const response = await axios.get(link); + const tweetUrl = response.request.res.responseUrl; + const match = regex.exec(tweetUrl); + if (Array.isArray(match) && typeof match[2] === 'string') { + const tweet = this._doc.createElement('div'); + tweet.innerText = 'Tweet placeholder'; + tweet.className = 'tweet-placeholder'; + tweet.setAttribute('data-tweet-id', match[2]); + element.parentNode.replaceChild(tweet, element); + } + } catch (e) { + this.log('Error loading tweet: ', link, e); + } } } @@ -2247,7 +2264,7 @@ Readability.prototype = { this._createInstagramPostPlaceholder(element, match[2]); } } - }); + } Array.from(e.getElementsByTagName('iframe')).forEach(element => { diff --git a/packages/readabilityjs/package.json b/packages/readabilityjs/package.json index ca3df23c8..32a65ff40 100644 --- a/packages/readabilityjs/package.json +++ b/packages/readabilityjs/package.json @@ -30,7 +30,8 @@ "mocha": "^8.2.0", "puppeteer": "^10.1.0", "sinon": "^7.3.2", - "linkedom": "^0.14.9" + "linkedom": "^0.14.9", + "axios": "^0.26.0" }, "dependencies": { "html-entities": "^2.3.2", diff --git a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json new file mode 100644 index 000000000..f41a6b334 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json @@ -0,0 +1,10 @@ +{ + "title": "The 2-minute 20-second video that changes everything", + "byline": "Michael Shellenberger", + "dir": null, + "excerpt": "For decades, people have claimed that homelessness is just a\n housing problem. Sure, many also have substance use and mental\n illness issues. But if we just give homeless people their own\n own studio apartments, and decriminalize public camping,\n drugs, and shoplifting, the problem will go away, many\n claimed.", + "siteName": null, + "publishedDate": "2001-05-25T16:00:00.000Z", + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html new file mode 100644 index 000000000..f11abe4e0 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html @@ -0,0 +1,38 @@ +
+
+
+
+
+ + + + + + + + +
+ +
+
+
+

For decades, people have claimed that homelessness is just a housing problem. Sure, many also have substance use and mental illness issues. But if we just give homeless people their own own studio apartments, and decriminalize public camping, drugs, and shoplifting, the problem will go away, many claimed.

+

That hasn’t happened. Instead, the open drug scenes have worsened. Nationally, drug overdoses and poisonings increased from 17,000 in 2000 to 108,000 in 2021. And California, which pioneered the “Housing First”/decriminalization approach saw its homeless population increase 31% between 2011 and 2020, even as homelessness declined 18% in the rest of the country.

+

I debunked the lies about homelessness in San Fransicko, in hundreds of articles, and on dozens of TV and podcast appearances. But when it comes to educating the public, nothing has been more impactful than the video interviews of homeless people that I’ve conducted over the last few months with my friend Leighton Woodhouse, a documentary filmmaker, as part of my run to become governor of California.

+

Now Leighton has assembled those interviews into a two-minute 20 second video we’ve posted on Twitter. It’s a must-watch. It’s only been on-line for a few hours, and over 130,000 people have seen it.

+

I hope you’ll take a minute to watch it. And, after you do, please consider a donation to Shellenberger for Governor.

+

There’s just 12 days before the primary election. Anyone can vote for anyone. And anyone in the US can donate. Whatever happens, we will make history.

+

+ Donate to Shellenberger 2022 +

+
+
+

+ You’re a free subscriber to Michael Shellenberger. For the full experience, become a paid subscriber. +

+

+ Subscribe +

+
+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/source.html b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/source.html new file mode 100644 index 000000000..8ac94637e --- /dev/null +++ b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/source.html @@ -0,0 +1,2000 @@ + + + The 2-minute 20-second video that changes everything + + + + +
+ For decades, people have claimed that homelessness is just a housing + problem. Sure, many also have substance use and mental illness issues. But + if we just give homeless people their own own studio apartments, and + decriminalize public camping, drugs, and shoplifting, the problem will go + away, many claimed. +  ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ +
+ + + + + + + + + + + + + + + diff --git a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/url.txt b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/url.txt new file mode 100644 index 000000000..2fa284abc --- /dev/null +++ b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/url.txt @@ -0,0 +1 @@ +https://michaelshellenberger.substack.com/p/the-2-minute-20-second-video-that?s=r \ No newline at end of file From 0b0edd3e690f6a64a864f3aa234552b0774a725e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 31 May 2022 22:50:14 +0800 Subject: [PATCH 07/10] Make fetching tweet url async --- packages/readabilityjs/Readability.js | 44 +++++++++++++-------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index f322a2113..60e67ce9c 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -830,8 +830,8 @@ Readability.prototype = { * @param Element * @return void **/ - _prepArticle: function (articleContent) { - this._createPlaceholders(articleContent); + _prepArticle: async function (articleContent) { + await this._createPlaceholders(articleContent); this._cleanStyles(articleContent); // Check for data tables before we continue, to avoid removing items in // those tables, which will often be isolated even though they're @@ -1105,7 +1105,7 @@ Readability.prototype = { * @param page a document to run upon. Needs to be a full document, complete with body. * @return Element **/ - _grabArticle: function (page) { + _grabArticle: async function(page) { this.log("**** grabArticle ****"); const doc = this._doc; const isPaging = page !== null; @@ -1148,13 +1148,13 @@ Readability.prototype = { if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) { const headingText = node.textContent.trim(); const titleText = this._articleTitle.trim(); - this.log("Removing header: ", {headingText, titleText}); + this.log("Removing header: ", { headingText, titleText }); shouldRemoveTitleHeader = false; // Replacing title with the heading if the title includes heading but heading is smaller // Example article: http://jsomers.net/i-should-have-loved-biology // Or if there is the specific attribute that we can lean on. // For example "headline" in this article - https://nymag.com/intelligencer/2020/12/four-seasons-total-landscaping-the-full-est-possible-story.html - if ((titleText !== headingText && titleText.includes(headingText)) || this._someNodeAttribute(node, ({value}) => value === 'headline')) { + if ((titleText !== headingText && titleText.includes(headingText)) || this._someNodeAttribute(node, ({ value }) => value === 'headline')) { this.log('Replacing title with heading') this._articleTitle = headingText; } @@ -1166,8 +1166,8 @@ Readability.prototype = { if (stripUnlikelyCandidates) { if ( (this.REGEXPS.unlikelyCandidates.test(matchString) || - // Checking for the "data-testid" attribute as well for the NYTimes articles - // Example article: https://www.nytimes.com/2021/03/31/world/americas/brazil-coronavirus-bolsonaro.html + // Checking for the "data-testid" attribute as well for the NYTimes articles + // Example article: https://www.nytimes.com/2021/03/31/world/americas/brazil-coronavirus-bolsonaro.html this.REGEXPS.unlikelyCandidates.test(node.dataset && node.dataset.testid)) && !this.REGEXPS.okMaybeItsACandidate.test(matchString) && !/tweet(-\w+)?/i.test(matchString) && @@ -1204,8 +1204,8 @@ Readability.prototype = { // Remove DIV, SECTION, and HEADER nodes without any content(e.g. text, image, video, or iframe). if ((node.tagName === "DIV" || node.tagName === "SECTION" || node.tagName === "HEADER" || - node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" || - node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") && + node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" || + node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") && this._isElementWithoutContent(node)) { node = this._removeAndGetNext(node); continue; @@ -1271,7 +1271,7 @@ Readability.prototype = { * A score is determined by things like number of commas, class names, etc. Maybe eventually link density. **/ var candidates = []; - this._forEachNode(elementsToScore, function (elementToScore) { + this._forEachNode(elementsToScore, function(elementToScore) { if (!elementToScore.parentNode || typeof (elementToScore.parentNode.tagName) === "undefined") return; @@ -1297,7 +1297,7 @@ Readability.prototype = { contentScore += Math.min(Math.floor(innerText.length / 100), 3); // Initialize and score ancestors. - this._forEachNode(ancestors, function (ancestor, level) { + this._forEachNode(ancestors, function(ancestor, level) { if (!ancestor.tagName || !ancestor.parentNode || typeof (ancestor.parentNode.tagName) === "undefined") return; @@ -1526,7 +1526,7 @@ Readability.prototype = { const figures = this._getAllNodesWithTag(headerNode, ['FIGURE']); this._forEachNode(figures, figure => { if (!this._someNode(alreadyExistingFigures, existingFigure => existingFigure === figure)) { - this.log(`Prepending figure to the article`, {className: figure.className, scr: figure.src}) + this.log(`Prepending figure to the article`, { className: figure.className, scr: figure.src }) articleContent.prepend(figure) } }) @@ -1535,7 +1535,7 @@ Readability.prototype = { if (this._debug) this.log("Article content pre-prep: ", { content: articleContent.innerHTML }); // So we have all of the content that we need. Now we clean it up for presentation. - this._prepArticle(articleContent); + await this._prepArticle(articleContent); if (this._debug) this.log("Article content post-prep: ", { content: articleContent.innerHTML }); @@ -1574,17 +1574,17 @@ Readability.prototype = { if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) { this._removeFlag(this.FLAG_STRIP_UNLIKELYS); - this._attempts.push({articleContent: articleContent, textLength: textLength}); + this._attempts.push({ articleContent: articleContent, textLength: textLength }); } else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) { this._removeFlag(this.FLAG_WEIGHT_CLASSES); - this._attempts.push({articleContent: articleContent, textLength: textLength}); + this._attempts.push({ articleContent: articleContent, textLength: textLength }); } else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) { this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY); - this._attempts.push({articleContent: articleContent, textLength: textLength}); + this._attempts.push({ articleContent: articleContent, textLength: textLength }); } else { - this._attempts.push({articleContent: articleContent, textLength: textLength}); + this._attempts.push({ articleContent: articleContent, textLength: textLength }); // No luck after removing flags, just return the longest text we found during the different loops - this._attempts.sort(function (a, b) { + this._attempts.sort(function(a, b) { return b.textLength - a.textLength; }); @@ -1601,7 +1601,7 @@ Readability.prototype = { if (parseSuccessful) { // Find out text direction from ancestors of final top candidate. var ancestors = [parentOfTopCandidate, topCandidate].concat(this._getNodeAncestors(parentOfTopCandidate)); - this._someNode(ancestors, function (ancestor) { + this._someNode(ancestors, function(ancestor) { if (!ancestor.tagName) return false; var articleDir = ancestor.getAttribute("dir"); @@ -2246,7 +2246,7 @@ Readability.prototype = { tweet.innerText = 'Tweet placeholder'; tweet.className = 'tweet-placeholder'; tweet.setAttribute('data-tweet-id', match[2]); - element.parentNode.replaceChild(tweet, element); + element.parentNode.replaceWith(tweet); } } catch (e) { this.log('Error loading tweet: ', link, e); @@ -2880,7 +2880,7 @@ Readability.prototype = { * * @return void **/ - parse: function () { + parse: async function() { // Avoid parsing too large documents, as per configuration option if (this._maxElemsToParse > 0) { var numTags = this._doc.getElementsByTagName("*").length; @@ -2905,7 +2905,7 @@ Readability.prototype = { var metadata = this._getArticleMetadata(jsonLd); this._articleTitle = metadata.title; - var articleContent = this._grabArticle(); + var articleContent = await this._grabArticle(); if (!articleContent) return null; From 404805e0c0af161967f9609577e81a34ef4ec596 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 31 May 2022 22:51:00 +0800 Subject: [PATCH 08/10] Make async calls to parse() --- packages/api/src/readability.d.ts | 2 +- packages/api/src/utils/parser.ts | 12 ++-- .../readabilityjs/test/generate-testcase.js | 12 ++-- .../readabilityjs/test/test-readability.js | 58 +++++++++---------- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index ba8278b14..2587b114e 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -72,7 +72,7 @@ declare module '@omnivore/readability' { * * The response will be null if the processing failed (https://github.com/mozilla/readability/blob/52ab9b5c8916c306a47b2119270dcdabebf9d203/Readability.js#L2038) */ - parse(): Readability.ParseResult | null + async parse(): Promise } namespace Readability { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index b397365f9..a7a08475b 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -133,12 +133,12 @@ const getPurifiedContent = (html: string): Document => { return parseHTML(clean).document } -const getReadabilityResult = ( +const getReadabilityResult = async ( url: string, html: string, document: Document, isNewsletter?: boolean -): Readability.ParseResult | null => { +): Promise => { // First attempt to read the article as is. // if that fails attempt to purify then read const sources = [ @@ -157,7 +157,7 @@ const getReadabilityResult = ( } try { - const article = new Readability(document, { + const article = await new Readability(document, { debug: DEBUG_MODE, createImageProxyUrl, keepTables: isNewsletter, @@ -236,7 +236,7 @@ export const parsePreparedContent = async ( await applyHandlers(url, dom) try { - article = getReadabilityResult(url, document, dom, isNewsletter) + article = await getReadabilityResult(url, document, dom, isNewsletter) if (!article?.textContent && allowRetry) { const newDocument = { ...preparedDocument, @@ -406,10 +406,10 @@ export const parseUrlMetadata = async ( // based on it's contents. // TODO: when we consolidate the handlers we could include this // as a utility method on each one. -export const isProbablyNewsletter = (html: string): boolean => { +export const isProbablyNewsletter = async (html: string): Promise => { const dom = parseHTML(html).document const domCopy = parseHTML(dom.documentElement.outerHTML).document - const article = new Readability(domCopy, { + const article = await new Readability(domCopy, { debug: false, keepTables: true, }).parse() diff --git a/packages/readabilityjs/test/generate-testcase.js b/packages/readabilityjs/test/generate-testcase.js index a757a1786..7658ac7da 100644 --- a/packages/readabilityjs/test/generate-testcase.js +++ b/packages/readabilityjs/test/generate-testcase.js @@ -192,7 +192,7 @@ function onResponseReceived(error, source, destRoot) { console.log("writing"); } var sourcePath = path.join(destRoot, "source.html"); - fs.writeFile(sourcePath, source, function (err) { + fs.writeFile(sourcePath, source, async function(err) { if (err) { console.error("Couldn't write data to source.html!"); console.error(err); @@ -201,11 +201,11 @@ function onResponseReceived(error, source, destRoot) { if (debug) { console.log("Running readability stuff"); } - runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json")); + await runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json")); }); } -function runReadability(source, destPath, metadataDestPath) { +async function runReadability(source, destPath, metadataDestPath) { var uri = "http://fakehost/test/page.html"; var myReader, result, readerable; try { @@ -215,7 +215,7 @@ function runReadability(source, destPath, metadataDestPath) { // We pass `caption` as a class to check that passing in extra classes works, // given that it appears in some of the test documents. myReader = new Readability(jsdom, { classesToPreserve: ["caption"], url: uri }); - result = myReader.parse(); + result = await myReader.parse(); } catch (ex) { console.error(ex); ex.stack.forEach(console.log.bind(console)); @@ -225,7 +225,7 @@ function runReadability(source, destPath, metadataDestPath) { return; } - fs.writeFile(destPath, prettyPrint(result.content), function (fileWriteErr) { + fs.writeFile(destPath, prettyPrint(result.content), function(fileWriteErr) { if (fileWriteErr) { console.error("Couldn't write data to expected.html!"); console.error(fileWriteErr); @@ -240,7 +240,7 @@ function runReadability(source, destPath, metadataDestPath) { // Add isProbablyReaderable result result.readerable = readerable; - fs.writeFile(metadataDestPath, JSON.stringify(result, null, 2) + "\n", function (metadataWriteErr) { + fs.writeFile(metadataDestPath, JSON.stringify(result, null, 2) + "\n", function(metadataWriteErr) { if (metadataWriteErr) { console.error("Couldn't write data to expected-metadata.json!"); console.error(metadataWriteErr); diff --git a/packages/readabilityjs/test/test-readability.js b/packages/readabilityjs/test/test-readability.js index 1228db774..14be0aa25 100644 --- a/packages/readabilityjs/test/test-readability.js +++ b/packages/readabilityjs/test/test-readability.js @@ -59,13 +59,13 @@ function runTestsWithItems(label, domGenerationFn, source, expectedContent, expe var result; - before(function() { + before(async function() { try { var doc = domGenerationFn(source); // Provide one class name to preserve, which we know appears in a few // of the test documents. var myReader = new Readability(doc, { classesToPreserve: ["caption"], url: uri }); - result = myReader.parse(); + result = await myReader.parse(); } catch (err) { throw reformatError(err); } @@ -222,68 +222,68 @@ describe("Readability API", function() { it("shouldn't parse oversized documents as per configuration", function() { var doc = new JSDOMParser().parse("
yo
"); - expect(function() { - new Readability(doc, {maxElemsToParse: 1}).parse(); + expect(async function() { + await (new Readability(doc, { maxElemsToParse: 1 })).parse(); }).to.Throw("Aborting parsing document; 2 elements found"); }); - it("should run _cleanClasses with default configuration", function() { + it("should run _cleanClasses with default configuration", async function() { var doc = parseHTML(exampleSource).document; var parser = new Readability(doc); parser._cleanClasses = sinon.fake(); - parser.parse(); + await parser.parse(); expect(parser._cleanClasses.called).eql(true); }); - it("should run _cleanClasses when option keepClasses = false", function() { + it("should run _cleanClasses when option keepClasses = false", async function() { var doc = parseHTML(exampleSource).document; - var parser = new Readability(doc, {keepClasses: false}); + var parser = new Readability(doc, { keepClasses: false }); parser._cleanClasses = sinon.fake(); - parser.parse(); + await parser.parse(); expect(parser._cleanClasses.called).eql(true); }); - it("shouldn't run _cleanClasses when option keepClasses = true", function() { + it("shouldn't run _cleanClasses when option keepClasses = true", async function() { var doc = parseHTML(exampleSource).document; - var parser = new Readability(doc, {keepClasses: true}); + var parser = new Readability(doc, { keepClasses: true }); parser._cleanClasses = sinon.fake(); - parser.parse(); + await parser.parse(); expect(parser._cleanClasses.called).eql(false); }); - xit("should use custom content serializer sent as option", function() { - var dom = new JSDOM("My cat: "); + xit("should use custom content serializer sent as option", async function() { + var dom = parseHTML("My cat: "); var expected_xhtml = "
My cat:
"; var xml = new dom.window.XMLSerializer(); - var content = new Readability(dom.window.document, { + var content = await (new Readability(dom.window.document, { serializer: function(el) { return xml.serializeToString(el.firstChild); } - }).parse().content; + })).parse().content; expect(content).eql(expected_xhtml); }); - it("should not proxy image with data uri", function() { + it("should not proxy image with data uri", async function() { var dom = parseHTML("My cat: \"Red"); var expected_xhtml = "
My cat: \"Red
"; - var content = new Readability(dom.document).parse().content; + var content = await (new Readability(dom.document)).parse().content; expect(content).eql(expected_xhtml); }); - it("should handle srcset elements with density descriptors", function() { + it("should handle srcset elements with density descriptors", async function() { var dom = parseHTML('My image: ' + ''); var expected_xhtml = '
My image: ' + - '
'; - var content = new Readability(dom.document, { + '
'; + var content = await (new Readability(dom.document, { createImageProxyUrl: function(url) { return url; } - }).parse().content; + })).parse().content; expect(content).eql(expected_xhtml); }); - it("should remove srcset elements that are lazy loading placeholders", function() { + it("should remove srcset elements that are lazy loading placeholders", async function() { var dom = parseHTML('My image: '); var expected_xhtml = '
' + 'My image: ' + '
'; - var content = new Readability(dom.document, { + var content = await (new Readability(dom.document, { createImageProxyUrl: function(url) { return url; } - }).parse().content; + })).parse().content; expect(content).eql(expected_xhtml); }); }); From 98ecdcff80193463106fa56a3b24a36cce6127c6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 31 May 2022 22:51:27 +0800 Subject: [PATCH 09/10] Fix test case of setting tweet-placeholder --- .../test/test-pages/substack-michaelshellenberger/expected.html | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html index f11abe4e0..f645b6fc0 100644 --- a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html +++ b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html @@ -20,6 +20,7 @@

That hasn’t happened. Instead, the open drug scenes have worsened. Nationally, drug overdoses and poisonings increased from 17,000 in 2000 to 108,000 in 2021. And California, which pioneered the “Housing First”/decriminalization approach saw its homeless population increase 31% between 2011 and 2020, even as homelessness declined 18% in the rest of the country.

I debunked the lies about homelessness in San Fransicko, in hundreds of articles, and on dozens of TV and podcast appearances. But when it comes to educating the public, nothing has been more impactful than the video interviews of homeless people that I’ve conducted over the last few months with my friend Leighton Woodhouse, a documentary filmmaker, as part of my run to become governor of California.

Now Leighton has assembled those interviews into a two-minute 20 second video we’ve posted on Twitter. It’s a must-watch. It’s only been on-line for a few hours, and over 130,000 people have seen it.

+

I hope you’ll take a minute to watch it. And, after you do, please consider a donation to Shellenberger for Governor.

There’s just 12 days before the primary election. Anyone can vote for anyone. And anyone in the US can donate. Whatever happens, we will make history.

From 304fe70113d9dd1791c000f18801a37ae6f99b79 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 1 Jun 2022 10:30:23 +0800 Subject: [PATCH 10/10] Fix tests --- packages/api/package.json | 2 + packages/api/src/routers/svc/emails.ts | 2 +- packages/api/test/utils/parser.test.ts | 104 +++++++++++------- packages/readabilityjs/package.json | 7 +- .../readabilityjs/test/test-readability.js | 20 ++-- yarn.lock | 36 ++++-- 6 files changed, 106 insertions(+), 65 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 37976687a..00f11fe3a 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -89,6 +89,7 @@ "@types/analytics-node": "^3.1.7", "@types/bcryptjs": "^2.4.2", "@types/chai": "^4.2.18", + "@types/chai-as-promised": "^7.1.5", "@types/chai-string": "^1.4.2", "@types/cookie": "^0.4.0", "@types/cookie-parser": "^1.4.2", @@ -109,6 +110,7 @@ "@types/uuid": "^8.3.0", "@types/voca": "^1.4.0", "chai": "^4.3.4", + "chai-as-promised": "^7.1.1", "chai-string": "^1.5.0", "circular-dependency-plugin": "^5.2.0", "mocha": "^9.0.1", diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts index 4bc000d81..3624cdeac 100644 --- a/packages/api/src/routers/svc/emails.ts +++ b/packages/api/src/routers/svc/emails.ts @@ -51,7 +51,7 @@ export function emailsServiceRouter() { return } - if (isProbablyNewsletter(data.html)) { + if (await isProbablyNewsletter(data.html)) { console.log('handling as newsletter', data) await saveNewsletterEmail({ email: data.to, diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts index 87c48e1b0..8c053ca8f 100644 --- a/packages/api/test/utils/parser.test.ts +++ b/packages/api/test/utils/parser.test.ts @@ -1,54 +1,73 @@ import 'mocha' +import * as chai from 'chai' import { expect } from 'chai' import 'chai/register-should' import fs from 'fs' -import { findNewsletterUrl, isProbablyNewsletter, parsePageMetadata, parsePreparedContent } from '../../src/utils/parser' +import { + findNewsletterUrl, + isProbablyNewsletter, + parsePageMetadata, + parsePreparedContent, +} from '../../src/utils/parser' import nock from 'nock' +import chaiAsPromised from 'chai-as-promised' + +chai.use(chaiAsPromised) const load = (path: string): string => { return fs.readFileSync(path, 'utf8') } describe('isProbablyNewsletter', () => { - it('returns true for substack newsletter', () => { + it('returns true for substack newsletter', async () => { const html = load('./test/utils/data/substack-forwarded-newsletter.html') - isProbablyNewsletter(html).should.be.true + await expect(isProbablyNewsletter(html)).to.eventually.be.true }) - it('returns true for private forwarded substack newsletter', () => { - const html = load('./test/utils/data/substack-private-forwarded-newsletter.html') - isProbablyNewsletter(html).should.be.true + it('returns true for private forwarded substack newsletter', async () => { + const html = load( + './test/utils/data/substack-private-forwarded-newsletter.html' + ) + await expect(isProbablyNewsletter(html)).to.eventually.be.true }) - it('returns false for substack welcome email', () => { + it('returns false for substack welcome email', async () => { const html = load('./test/utils/data/substack-forwarded-welcome-email.html') - isProbablyNewsletter(html).should.be.false + await expect(isProbablyNewsletter(html)).to.eventually.be.false }) - it('returns true for beehiiv.com newsletter', () => { + it('returns true for beehiiv.com newsletter', async () => { const html = load('./test/utils/data/beehiiv-newsletter.html') - isProbablyNewsletter(html).should.be.true + await expect(isProbablyNewsletter(html)).to.eventually.be.true }) }) describe('findNewsletterUrl', async () => { it('gets the URL from the header if it is a substack newsletter', async () => { nock('https://newsletter.slowchinese.net') - .head('/p/companies-that-eat-people-217?token=eyJ1c2VyX2lkIjoxMTU0MzM0NSwicG9zdF9pZCI6NDg3MjA5NDAsImlhdCI6MTY0NTI1NzQ1MSwiaXNzIjoicHViLTI4MDUzMSIsInN1YiI6InBvc3QtcmVhY3Rpb24ifQ.l5F3Kx6K9tvy9cRAXx3MepobQBCJDJQgAxOpA0INIZA') - .reply(200, ''); + .head( + '/p/companies-that-eat-people-217?token=eyJ1c2VyX2lkIjoxMTU0MzM0NSwicG9zdF9pZCI6NDg3MjA5NDAsImlhdCI6MTY0NTI1NzQ1MSwiaXNzIjoicHViLTI4MDUzMSIsInN1YiI6InBvc3QtcmVhY3Rpb24ifQ.l5F3Kx6K9tvy9cRAXx3MepobQBCJDJQgAxOpA0INIZA' + ) + .reply(200, '') const html = load('./test/utils/data/substack-forwarded-newsletter.html') const url = await findNewsletterUrl(html) // Not sure if the redirects from substack expire, this test could eventually fail - expect(url).to.startWith('https://newsletter.slowchinese.net/p/companies-that-eat-people-217') + expect(url).to.startWith( + 'https://newsletter.slowchinese.net/p/companies-that-eat-people-217' + ) }) it('gets the URL from the header if it is a beehiiv newsletter', async () => { nock('https://u23463625.ct.sendgrid.net') - .head('/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno') - .reply(302, undefined,{ - 'Location': 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple' - }) - .get('/p/talked-guy-spent-30m-beeple') - .reply(200, ''); + .head( + '/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno' + ) + .reply(302, undefined, { + Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple', + }) + .get('/p/talked-guy-spent-30m-beeple') + .reply(200, '') const html = load('./test/utils/data/beehiiv-newsletter.html') const url = await findNewsletterUrl(html) - expect(url).to.startWith('https://www.milkroad.com/p/talked-guy-spent-30m-beeple') + expect(url).to.startWith( + 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple' + ) }) it('returns undefined if it is not a newsletter', async () => { const html = load('./test/utils/data/substack-forwarded-welcome-email.html') @@ -63,31 +82,35 @@ describe('parseMetadata', async () => { const metadata = await parsePageMetadata(html) expect(metadata?.author).to.deep.equal('Omnivore') expect(metadata?.title).to.deep.equal('Code Block Syntax Highlighting') - expect(metadata?.previewImage).to.deep.equal('https://cdn.substack.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F2ab1f7e8-2ca7-4011-8ccb-43d0b3bd244f_1490x2020.png') - expect(metadata?.description).to.deep.equal('Highlighted in Omnivore') + expect(metadata?.previewImage).to.deep.equal( + 'https://cdn.substack.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F2ab1f7e8-2ca7-4011-8ccb-43d0b3bd244f_1490x2020.png' + ) + expect(metadata?.description).to.deep.equal( + 'Highlighted in Omnivore' + ) }) }) describe('parsePreparedContent', async () => { it('gets published date when JSONLD fails to load', async () => { const html = load('./test/utils/data/stratechery-blog-post.html') - const result = await parsePreparedContent( - 'https://example.com/', - { - document: html, - pageInfo: { } - }, + const result = await parsePreparedContent('https://example.com/', { + document: html, + pageInfo: {}, + }) + expect(result.parsedContent?.publishedDate?.getTime()).to.equal( + new Date('2016-04-05T15:27:51+00:00').getTime() ) - expect(result.parsedContent?.publishedDate?.getTime()).to.equal(new Date('2016-04-05T15:27:51+00:00').getTime()) }) }) describe('parsePreparedContent', async () => { nock('https://oembeddata').get('/').reply(200, { - "version":"1.0", - "provider_name":"Hippocratic Adventures", - "provider_url":"https:\/\/www.hippocraticadventures.com", - "title":"The Ultimate Guide to Practicing Medicine in Singapore – Part 2" + version: '1.0', + provider_name: 'Hippocratic Adventures', + provider_url: 'https://www.hippocraticadventures.com', + title: + 'The Ultimate Guide to Practicing Medicine in Singapore – Part 2', }) it('gets metadata from external JSONLD if available', async () => { @@ -98,13 +121,12 @@ describe('parsePreparedContent', async () => { body ` - const result = await parsePreparedContent( - 'https://example.com/', - { - document: html, - pageInfo: { } - }, - ) - expect(result.parsedContent?.title).to.equal('The Ultimate Guide to Practicing Medicine in Singapore – Part 2') + const result = await parsePreparedContent('https://example.com/', { + document: html, + pageInfo: {}, + }) + expect(result.parsedContent?.title).to.equal( + 'The Ultimate Guide to Practicing Medicine in Singapore – Part 2' + ) }) }) diff --git a/packages/readabilityjs/package.json b/packages/readabilityjs/package.json index 32a65ff40..227de2c2a 100644 --- a/packages/readabilityjs/package.json +++ b/packages/readabilityjs/package.json @@ -24,14 +24,15 @@ "homepage": "https://github.com/mozilla/readability", "devDependencies": { "@c4312/matcha": "^1.3.1", + "axios": "^0.26.0", "chai": "^2.1.*", + "chai-as-promised": "^7.1.1", "htmltidy2": "^0.3.0", "js-beautify": "^1.13.0", + "linkedom": "^0.14.9", "mocha": "^8.2.0", "puppeteer": "^10.1.0", - "sinon": "^7.3.2", - "linkedom": "^0.14.9", - "axios": "^0.26.0" + "sinon": "^7.3.2" }, "dependencies": { "html-entities": "^2.3.2", diff --git a/packages/readabilityjs/test/test-readability.js b/packages/readabilityjs/test/test-readability.js index 14be0aa25..6db89071e 100644 --- a/packages/readabilityjs/test/test-readability.js +++ b/packages/readabilityjs/test/test-readability.js @@ -1,7 +1,9 @@ var chai = require("chai"); var sinon = require("sinon"); +var chaiAsPromised = require("chai-as-promised"); const { parseHTML } = require("linkedom"); +chai.use(chaiAsPromised); chai.config.includeStack = true; var expect = chai.expect; @@ -220,11 +222,11 @@ describe("Readability API", function() { describe("#parse", function() { var exampleSource = testPages[0].source; - it("shouldn't parse oversized documents as per configuration", function() { + it("shouldn't parse oversized documents as per configuration", async function() { var doc = new JSDOMParser().parse("

yo
"); - expect(async function() { - await (new Readability(doc, { maxElemsToParse: 1 })).parse(); - }).to.Throw("Aborting parsing document; 2 elements found"); + await expect( + (new Readability(doc, { maxElemsToParse: 1 })).parse() + ).to.be.rejectedWith("Aborting parsing document; 2 elements found"); }); it("should run _cleanClasses with default configuration", async function() { @@ -279,7 +281,7 @@ describe("Readability API", function() { var expected_xhtml = "
My cat: \"Red
"; - var content = await (new Readability(dom.document)).parse().content; + var content = (await (new Readability(dom.document)).parse()).content; expect(content).eql(expected_xhtml); }); @@ -296,11 +298,11 @@ describe("Readability API", function() { 'https://webkit.org/demos/srcset/image-2x.png 2x,' + 'https://webkit.org/demos/srcset/image-3x.png 3x,' + 'https://webkit.org/demos/srcset/image-4x.png 4x,">
'; - var content = await (new Readability(dom.document, { + var content = (await (new Readability(dom.document, { createImageProxyUrl: function(url) { return url; } - })).parse().content; + })).parse()).content; expect(content).eql(expected_xhtml); }); @@ -309,11 +311,11 @@ describe("Readability API", function() { var expected_xhtml = '
' + 'My image: ' + '
'; - var content = await (new Readability(dom.document, { + var content = (await (new Readability(dom.document, { createImageProxyUrl: function(url) { return url; } - })).parse().content; + })).parse()).content; expect(content).eql(expected_xhtml); }); }); diff --git a/yarn.lock b/yarn.lock index d516c3c6f..2f5cc13e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7382,6 +7382,13 @@ resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.2.tgz#f65d3d6389e01eeb458bd54dc8f52b95a9463bc8" integrity sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w== +"@types/chai-as-promised@^7.1.5": + version "7.1.5" + resolved "https://registry.yarnpkg.com/@types/chai-as-promised/-/chai-as-promised-7.1.5.tgz#6e016811f6c7a64f2eed823191c3a6955094e255" + integrity sha512-jStwss93SITGBwt/niYrkf2C+/1KTeZCZl1LaeezTlqppAKeoQC7jxyqYuP72sxBGKCIbw7oHgbYssIRzT5FCQ== + dependencies: + "@types/chai" "*" + "@types/chai-string@^1.4.2": version "1.4.2" resolved "https://registry.yarnpkg.com/@types/chai-string/-/chai-string-1.4.2.tgz#0f116504a666b6c6a3c42becf86634316c9a19ac" @@ -10321,6 +10328,13 @@ ccount@^1.0.0: resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.1.0.tgz#246687debb6014735131be8abab2d93898f8d043" integrity sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg== +chai-as-promised@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/chai-as-promised/-/chai-as-promised-7.1.1.tgz#08645d825deb8696ee61725dbf590c012eb00ca0" + integrity sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA== + dependencies: + check-error "^1.0.2" + chai-string@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/chai-string/-/chai-string-1.5.0.tgz#0bdb2d8a5f1dbe90bc78ec493c1c1c180dd4d3d2" @@ -17133,7 +17147,7 @@ lambdafs@^2.0.3: resolved "https://registry.yarnpkg.com/lambdafs/-/lambdafs-2.1.1.tgz#4bf8d3037b6c61bbb4a22ab05c73ee47964c25ed" integrity sha512-x5k8JcoJWkWLvCVBzrl4pzvkEHSgSBqFjg3Dpsc4AcTMq7oUMym4cL/gRTZ6VM4mUMY+M0dIbQ+V1c1tsqqanQ== dependencies: - tar-fs "*" + tar-fs "^2.1.1" language-subtag-registry@~0.3.2: version "0.3.21" @@ -23139,16 +23153,6 @@ tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -tar-fs@*, tar-fs@2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" - integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== - dependencies: - chownr "^1.1.1" - mkdirp-classic "^0.5.2" - pump "^3.0.0" - tar-stream "^2.1.4" - tar-fs@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.0.tgz#677700fc0c8b337a78bee3623fdc235f21d7afad" @@ -23159,6 +23163,16 @@ tar-fs@2.0.0: pump "^3.0.0" tar-stream "^2.0.0" +tar-fs@2.1.1, tar-fs@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" + integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== + dependencies: + chownr "^1.1.1" + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^2.1.4" + tar-stream@^2.0.0, tar-stream@^2.1.4: version "2.2.0" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"