From 1a1617e86c252da0778f08abf86e34f9c049c266 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 10 Feb 2023 17:13:18 +0800 Subject: [PATCH 1/7] Add readability test for habr.com --- .../habr.com/expected-metadata.json | 12 + .../test/test-pages/habr.com/expected.html | 277 +++ .../test/test-pages/habr.com/source.html | 1776 +++++++++++++++++ .../test/test-pages/habr.com/url.txt | 1 + 4 files changed, 2066 insertions(+) create mode 100644 packages/readabilityjs/test/test-pages/habr.com/expected-metadata.json create mode 100644 packages/readabilityjs/test/test-pages/habr.com/expected.html create mode 100644 packages/readabilityjs/test/test-pages/habr.com/source.html create mode 100644 packages/readabilityjs/test/test-pages/habr.com/url.txt diff --git a/packages/readabilityjs/test/test-pages/habr.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/habr.com/expected-metadata.json new file mode 100644 index 000000000..b3d693b9c --- /dev/null +++ b/packages/readabilityjs/test/test-pages/habr.com/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "Js, трюки, наблюдения, бенчмарки и как Лиса уничтожает Хром. Я протестировал всё, что вам было лень", + "byline": "programmerguru", + "dir": null, + "excerpt": "Картинка, конечно, стронгли анрилейтед Разные трюки я тестировал на Google Chrome 107.0.5304.107 и Mozilla Firefox 107.0 на Windows 10. Чтобы результаты всегда...", + "siteName": "Habr", + "siteIcon": "https://assets.habr.com/habr-web/img/favicons/favicon-16.png", + "previewImage": "https://habr.com/share/publication/712386/ff74768c013a8fb6236b5cbace64588a/", + "publishedDate": "2023-01-24T09:00:03.000Z", + "language": "Russian", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/habr.com/expected.html b/packages/readabilityjs/test/test-pages/habr.com/expected.html new file mode 100644 index 000000000..d65667971 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/habr.com/expected.html @@ -0,0 +1,277 @@ +
+
+ +
+ +

+

+ +

Картинка, конечно, стронгли анрилейтед

+

Разные трюки я тестировал на Google Chrome 107.0.5304.107 и Mozilla Firefox 107.0 на Windows 10.

+

Чтобы результаты всегда были железно воспроизводимыми, я отключил все С-State’ы, ядра зафиксировал на 5 ГГц.

+

У меня 9900К, это Coffee Lake c AVX256, какие оптимизации применит Jit для вашего процессора — я не знаю, результат на вашем компьютере может отличаться от моего, в т.ч. из-за микроархитектуры процессора.

+

Скорость парсинга кода тоже входит в бенчмарк, поэтому браузер с быстрым парсером будет впереди.
+ +

+

+ Есть ли у переменной оверхед? +

+

Есть ли смысл использовать только dot notation? Какова цена выноса лишней переменной?

+ 
var array = new Array(65535).fill()
+
+// 3
+var a = array.map((x) => x)
+var b = array.map((x) => x)
+var c = array.map((x) => x)
+
+// 2
+var a = array.map((x) => x)
+var b = array.map((x) => x).map((x) => x)
+
+// 1
+var a = array.map((x) => x).map((x) => x).map((x) => x)
+

Чтобы узнать, гоняет ли браузер память туда-сюда, делаем мы .map на массив длиной 65535 с нулями внутри. Линк.

+

Здесь и далее в качестве единицы измерения на графиках будет указано кол-во выполнений кода в секунду, включая парсинг и компиляцию. Больше — лучше

+

Хром не заметил разницы, а вот лиса заметила. Применительно к лисе, у лишней переменной есть измеряемый оверхед.

+

+ Есть ли разница между var, let, const или их отсутствием? +


+

Проверим. Используя разные биндинги — создадим POJO с переменной е. Потом добавим ему функцию о и запустим эту функцию. Бенчмарк простой, но движущихся частей много. Линк.

+ 
var g = { e: [] }
+g.o = function(x) { g.e.push(...[1,2,3]) }
+g.o()
+

Код выглядит так, отличаются только биндинги.

+

Результат неожиданный, но железно воспроизводимый. var, быстрее.

+

+ Bounce pattern, Switch case, длинная тернарка +

+

Если обе конструкции логически одинаковые, они должны строить одно и то же синтаксическое дерево, верно? Давайте проверим.

+ 
// switch case
+function thing(e) {
+    switch (e) {
+      case 0:
+        return "0";
+
+      case 1:
+        return "1";
+
+      case 2:
+        return "2";
+
+      case 3:
+        return "3";
+        
+      default:
+        return "";
+    }
+}
+
+// bounce pattern
+function bounce(x)
+{
+   if (x === 0) return "0";
+   if (x === 1) return "1";
+   if (x === 2) return "2";
+   if (x === 3) return "3";
+   
+   return ""
+}
+
+// ternary
+function bounce(x) {
+  return 0 === x ? "0" : 1 === x ? "1" : 2 === x ? "2" : 3 === x ? "3" : "";
+}
+

Вот так выглядит код. Для всех вариантов он одинаков, отличаются только вызовы.

+

+ ▍ 1. Вызов в цикле +


+ 
for (let t = 0; 1e5 > t; t++) bounce(0), bounce(2), bounce(6);
+

Вызов выглядит так. Линк.


+

+ ▍ 2. В цикле с другим типом +


+ 
for (let t = 0; 1e5 > t; t++) bounce("0"), bounce("2"), bounce("");
+

Тут мы покидываем строку вместо числа. В свитче и if блоках используется строгое равенство, поэтому свитч выходит только через default, а if’ы выходят только последний return. Линк.


+

+ ▍ 3. Без цикла +


+ 
bounce(0), bounce(2), bounce(6)
+

Просто три вызова подряд, никаких циклов. Линк.

+

Похоже, что после первоначальной компиляции лиса не пытается дальше оптимизировать цикл, как это делает хром.

+

Также лиса, похоже, не строит одно и то же AST, как это делает хром. Рекомендую заменить ваши длинные if’ы и bounce паттерны на свитчи, чтобы избежать лисиных тормозов.

+

+ Инициализация массива +

+

Для примера возьму из паттернов функционального программирования, когда ты инициализируешь массив, прокидывая лямбду в инициализатор. Просто ради примера, в качестве этой лямбды будет fizzbuzz.

+ 
var times = 65535;
+
+function initializer(val, z) {
+    const i = z % 5 | 0;
+    return 0 == (z % 3 | 0) ? 0 === i ? "fizzbuzz" : "fizz" : 0 === i ? "buzz" : z;
+}
+
+// for i
+var b = new Array(times);
+for (var i = 0; i < times; i++) {
+    b[i] = initializer(b[i], i)
+}
+b
+
+// for push
+var c = [];
+for (var i = 0; i < times; i++) {
+    c.push(initializer(c[i], i))
+}
+c
+
+// Fill Map
+new Array(times).fill().map(initializer)
+
+

Это не самый красивый fizzbuzz, но это мой fizzbuzz. Линк на бенч.

+

Вариант с fill map создаёт два массива, сначала при вызове конструктора, потом при вызове map. Но такой вариант безальтернативно быстрее на хроме.

+

+ Конкатенация массивов +


+
+ 
// reduce
+arr.reduce((acc, val) => acc.concat(val), [])
+
+// flatMap
+arr.flatMap(x => x)
+
+// flat
+arr.flat()
+
+// reduce push
+arr.reduce((acc, val) => {
+    if (val) val.forEach(a => acc.push(a));
+    return acc;
+}, [])
+
+// forEach push
+let acc = [];
+
+arr.forEach(val => {
+    val && val.forEach(v => acc.push(v));
+}), acc;
+
+//concat spread
+[].concat(...arr)
+
+

Конкатенация массивов на 1 уровень, поведение идентичное flat(1). Линк.

+

Иногда я не понимаю, почему разработчики движков оставили такой потенциал для оптимизации.

+

+ Уничтожение хрома +

+

Бенчмарки ниже я перепроверял по нескольку раз, результат одинаковый и верный. Лиса действительно такая быстрая.

+

+ ▍ Итерация по массиву +

+

Сравнивать будем Array.prototype.forEach vs for...of vs for. На код смотрите по линку.

+

Ради производительности, циклы for, лучше переделать в forEach, чтобы хром не отставал.

+

+ ▍ Содержит ли строка значение +


+ 
// text.includes()
+url.includes('matchthis')
+
+// text.test()
+/matchthis/.test(url)
+
+// text.match()
+url.match(/matchthis/).length >= 0
+
+// text.indexOf()
+url.indexOf('matchthis') >= 0
+
+// text.search()
+url.search('matchthis') >= 0
+
+

Трюк с IndexOf быстрее и на лисе, и на хроме. Используйте трюк с IndexOf. Линк на бенчмарк.

+

+ Преобразование строки в число +

+

Тестируем неявное преобразование, парсинг и вызов конструктора.

+ 
// implicit
+var imp = + strNum
+
+// parseFloat
+var toStr = parseFloat(strNum)
+
+//Number
+var num = Number(strNum)
+

+

+ ▍ Int +

+

Линк на бенч.

+

+ ▍ Float +

+

Я перепроверял, это не ошибка. Неявный каст стринги в инт практически бесплатный у лисы. Линк на бенч.

+

+ Выводы +


+
    +
  1. Лисичка похорошела.
    2. +
  2. JS сделан за неделю на коленке.
    3. +
  3. Я не пишу на JS.
    4. +
  4. Вы тоже прекращайте.
    5. +

+
+ Играй в нашу новую игру прямо в Telegram! +
+

+

+
+ +
+
+

Теги:

+ +
+
+

Хабы:

+ +
+
+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/habr.com/source.html b/packages/readabilityjs/test/test-pages/habr.com/source.html new file mode 100644 index 000000000..689fbeb90 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/habr.com/source.html @@ -0,0 +1,1776 @@ + + + + + + + + Js, трюки, наблюдения, бенчмарки и как Лиса уничтожает Хром. Я протестировал всё, что вам было лень / Хабр + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+
+ +
+
+ +
+
Как стать автором +
+ Зарплаты айтишников +
Подкаст про дзен между разработкой и продуктом +
+
+
+
+
+
+
+
+
+
+ +
+
+
+ + + Поиск + + +
+
+
+ + + Профиль + + +
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+ +
+
+ + + Обновить + + +
+
+
+
+
+
+ +
+ +
+
+
+
+ 2231.74 +
+
+
+ Рейтинг +
+
+
+
+ RUVDS.com +
+ VDS/VPS-хостинг. Скидка 10% по коду HABR10 +
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+

+ Js, трюки, наблюдения, бенчмарки и как Лиса уничтожает Хром. Я протестировал всё, что вам было лень +

+
+ +
+ + + Время прочтения + + 4 мин +
+ + Просмотры + + 22K +
+
+
+ Блог компании RUVDS.com Высокая производительность *JavaScript *Клиентская оптимизация *Браузеры +
+
+
+
+
+
+
+
+
+ +
+ +
Картинка, конечно, стронгли анрилейтед
+
+ Разные трюки я тестировал на Google Chrome 107.0.5304.107 и Mozilla Firefox 107.0 на Windows 10.
+
+ Чтобы результаты всегда были железно воспроизводимыми, я отключил все С-State’ы, ядра зафиксировал на 5 ГГц.
+
+ У меня 9900К, это Coffee Lake c AVX256, какие оптимизации применит Jit для вашего процессора — я не знаю, результат на вашем компьютере может отличаться от моего, в т.ч. из-за микроархитектуры процессора.
+
+ Скорость парсинга кода тоже входит в бенчмарк, поэтому браузер с быстрым парсером будет впереди.
+
+

+ Есть ли у переменной оверхед? +


+ Есть ли смысл использовать только dot notation? Какова цена выноса лишней переменной?
+
+ 
var array = new Array(65535).fill()
+
+// 3
+var a = array.map((x) => x)
+var b = array.map((x) => x)
+var c = array.map((x) => x)
+
+// 2
+var a = array.map((x) => x)
+var b = array.map((x) => x).map((x) => x)
+
+// 1
+var a = array.map((x) => x).map((x) => x).map((x) => x)

+ Чтобы узнать, гоняет ли браузер память туда-сюда, делаем мы .map на массив длиной 65535 с нулями внутри. Линк.
+
+
Здесь и далее в качестве единицы измерения на графиках будет указано кол-во выполнений кода в секунду, включая парсинг и компиляцию. Больше — лучше
+
+ Хром не заметил разницы, а вот лиса заметила. Применительно к лисе, у лишней переменной есть измеряемый оверхед.
+
+

+ Есть ли разница между var, let, const или их отсутствием? +


+

+ Проверим. Используя разные биндинги — создадим POJO с переменной е. Потом добавим ему функцию о и запустим эту функцию. Бенчмарк простой, но движущихся частей много. Линк.
+
+ 
var g = { e: [] }
+g.o = function(x) { g.e.push(...[1,2,3]) }
+g.o()

+ Код выглядит так, отличаются только биндинги.
+
+

+ Результат неожиданный, но железно воспроизводимый. var, быстрее.
+
+

+ Bounce pattern, Switch case, длинная тернарка +


+ Если обе конструкции логически одинаковые, они должны строить одно и то же синтаксическое дерево, верно? Давайте проверим.
+
+ 
// switch case
+function thing(e) {
+    switch (e) {
+      case 0:
+        return "0";
+
+      case 1:
+        return "1";
+
+      case 2:
+        return "2";
+
+      case 3:
+        return "3";
+        
+      default:
+        return "";
+    }
+}
+
+// bounce pattern
+function bounce(x)
+{
+   if (x === 0) return "0";
+   if (x === 1) return "1";
+   if (x === 2) return "2";
+   if (x === 3) return "3";
+   
+   return ""
+}
+
+// ternary
+function bounce(x) {
+  return 0 === x ? "0" : 1 === x ? "1" : 2 === x ? "2" : 3 === x ? "3" : "";
+}

+ Вот так выглядит код. Для всех вариантов он одинаков, отличаются только вызовы.
+
+

+ ▍ 1. Вызов в цикле +


+ 
for (let t = 0; 1e5 > t; t++) bounce(0), bounce(2), bounce(6);

+ Вызов выглядит так. Линк.
+
+

+

+ ▍ 2. В цикле с другим типом +


+ 
for (let t = 0; 1e5 > t; t++) bounce("0"), bounce("2"), bounce("");

+ Тут мы покидываем строку вместо числа. В свитче и if блоках используется строгое равенство, поэтому свитч выходит только через default, а if’ы выходят только последний return. Линк.
+
+

+

+ ▍ 3. Без цикла +


+ 
bounce(0), bounce(2), bounce(6)

+ Просто три вызова подряд, никаких циклов. Линк.
+
+

+ Похоже, что после первоначальной компиляции лиса не пытается дальше оптимизировать цикл, как это делает хром.
+
+ Также лиса, похоже, не строит одно и то же AST, как это делает хром. Рекомендую заменить ваши длинные if’ы и bounce паттерны на свитчи, чтобы избежать лисиных тормозов.
+
+

+ Инициализация массива +


+ Для примера возьму из паттернов функционального программирования, когда ты инициализируешь массив, прокидывая лямбду в инициализатор. Просто ради примера, в качестве этой лямбды будет fizzbuzz.
+
+ 
var times = 65535;
+
+function initializer(val, z) {
+    const i = z % 5 | 0;
+    return 0 == (z % 3 | 0) ? 0 === i ? "fizzbuzz" : "fizz" : 0 === i ? "buzz" : z;
+}
+
+// for i
+var b = new Array(times);
+for (var i = 0; i < times; i++) {
+    b[i] = initializer(b[i], i)
+}
+b
+
+// for push
+var c = [];
+for (var i = 0; i < times; i++) {
+    c.push(initializer(c[i], i))
+}
+c
+
+// Fill Map
+new Array(times).fill().map(initializer)
+

+ Это не самый красивый fizzbuzz, но это мой fizzbuzz. Линк на бенч.
+
+

+ Вариант с fill map создаёт два массива, сначала при вызове конструктора, потом при вызове map. Но такой вариант безальтернативно быстрее на хроме.
+
+

+ Конкатенация массивов +


+

+ 
// reduce
+arr.reduce((acc, val) => acc.concat(val), [])
+
+// flatMap
+arr.flatMap(x => x)
+
+// flat
+arr.flat()
+
+// reduce push
+arr.reduce((acc, val) => {
+    if (val) val.forEach(a => acc.push(a));
+    return acc;
+}, [])
+
+// forEach push
+let acc = [];
+
+arr.forEach(val => {
+    val && val.forEach(v => acc.push(v));
+}), acc;
+
+//concat spread
+[].concat(...arr)
+

+ Конкатенация массивов на 1 уровень, поведение идентичное flat(1). Линк.
+
+

+ Иногда я не понимаю, почему разработчики движков оставили такой потенциал для оптимизации.
+
+

+ Уничтожение хрома +


+ Бенчмарки ниже я перепроверял по нескольку раз, результат одинаковый и верный. Лиса действительно такая быстрая.
+
+

+ ▍ Итерация по массиву +


+ Сравнивать будем Array.prototype.forEach vs for...of vs for. На код смотрите по линку.
+
+

+ Ради производительности, циклы for, лучше переделать в forEach, чтобы хром не отставал.
+
+

+ ▍ Содержит ли строка значение +


+ 
// text.includes()
+url.includes('matchthis')
+
+// text.test()
+/matchthis/.test(url)
+
+// text.match()
+url.match(/matchthis/).length >= 0
+
+// text.indexOf()
+url.indexOf('matchthis') >= 0
+
+// text.search()
+url.search('matchthis') >= 0
+

+
+

+ Трюк с IndexOf быстрее и на лисе, и на хроме. Используйте трюк с IndexOf. Линк на бенчмарк.
+
+

+ Преобразование строки в число +


+ Тестируем неявное преобразование, парсинг и вызов конструктора.
+
+ 
// implicit
+var imp = + strNum
+
+// parseFloat
+var toStr = parseFloat(strNum)
+
+//Number
+var num = Number(strNum)
+

+

+ ▍ Int +


+
+

+ Линк на бенч.
+
+

+ ▍ Float +


+
+

+ Я перепроверял, это не ошибка. Неявный каст стринги в инт практически бесплатный у лисы. Линк на бенч.
+
+

+ Выводы +


+
    +
  1. Лисичка похорошела. +
    2. +
  2. JS сделан за неделю на коленке. +
    3. +
  3. Я не пишу на JS. +
    4. +
  4. Вы тоже прекращайте. +
    5. +

+
+ Играй в нашу новую игру прямо в Telegram! +
+
+
+
+ + +
+ +
+
+
+ Теги: + +
+
+ Хабы: + +
+
+
+
+
+
+
+
+
+ + + Всего голосов 132: ↑116 и ↓16 + + +100 +
+ +
+
+ + + Комментарии + + 77 +77 +
+ + +
+
+
+ +
+
+
+ + + Закрыть + + +

+ Редакторский дайджест +

+

+ Присылаем лучшие статьи раз в месяц +

+
+
+ +
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+ +
+
+ RUVDS.com +
+ VDS/VPS-хостинг. Скидка 10% по коду HABR10 +
+
+
+
+ +
+
+
+ Telegram ВКонтакте Twitter +
+
+
+
+
+
+
+ +
+ +
+
+
+
+ 51 +
+
+ Карма +
+ +
+
+
+
+ 96.1 +
+
+
+ Рейтинг +
+
+
+
+
+
+
+ @programmerguru +
+

+ Пользователь +

+
+
+
+ +
+
+
+ +
+
+
+
+
+
+
+
+
+

+ Комментарии 77 +

+
+ +
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+

+ Публикации +

+
+
+
+
+
+ +
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+

+ Информация +

+
+
+
+
+
+
+ Сайт +
+
+ ruvds.com +
+
+
+
+ Дата регистрации +
+
+ +
+
+
+
+ Дата основания +
+
+ +
+
+
+
+ Численность +
+
+ 11–30 человек +
+
+
+
+ Местоположение +
+
+ Россия +
+
+
+
+ Представитель +
+
+ ruvds +
+
+
+
+
+
+
+
+
+

+ Ссылки +

+
+
+
+ +
+
+
+
+
+

+ Приложения +

+
+
+
+
    +
  • +
    + +
    +

    + RUVDS Client +

    +

    + Приложение для мониторинга и управления виртуальными серверами RUVDS с мобильных устройств. +

    + +
    +
    +
    • +
+
+
+
+
+
+

+ Виджет +

+
+
+
+
+ +
+
+
+
+
+
+

+ Виджет +

+
+
+
+
+ +
+
+
+
+
+
+
+

+ Блог на Хабре +

+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Ваш аккаунт +

+
+ +
+
+
+

+ Разделы +

+
+ +
+
+
+

+ Информация +

+
+ +
+
+
+

+ Услуги +

+
+ +
+
+
+
+
+
+
+
+ + + Техническая поддержка Вернуться на старую версию +
+ © 2006–2023, Habr +
+
+
+
+
+
+ +
+
+ + + + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ + diff --git a/packages/readabilityjs/test/test-pages/habr.com/url.txt b/packages/readabilityjs/test/test-pages/habr.com/url.txt new file mode 100644 index 000000000..082f687a1 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/habr.com/url.txt @@ -0,0 +1 @@ +https://habr.com/ru/company/ruvds/blog/712386/ \ No newline at end of file From d816ee9563ce345780506a4f822bf5a5b57dd00d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 13 Feb 2023 16:50:39 +0800 Subject: [PATCH 2/7] Allow image and media to be loaded in puppeteer --- .../src/services/create_page_save_request.ts | 2 +- packages/api/src/utils/parser.ts | 6 ++-- packages/puppeteer-parse/index.js | 31 ++----------------- 3 files changed, 6 insertions(+), 33 deletions(-) diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 60a7e5672..b0d134fef 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -93,7 +93,7 @@ export const createPageSaveRequest = async ( url: normalizedUrl, }) if (page) { - console.log('Page already exists', page) + console.log('Page already exists', page.id, page.url) articleSavingRequestId = page.id } else { page = { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 93c9eaffc..b1d8a1e8a 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -203,7 +203,7 @@ export const parsePreparedContent = async ( return { canonicalUrl: url, parsedContent: null, - domContent: preparedDocument.document, + domContent: document, pageType: PageType.Unknown, } } @@ -223,7 +223,7 @@ export const parsePreparedContent = async ( if (!article?.textContent && allowRetry) { const newDocument = { ...preparedDocument, - document: '' + preparedDocument.document + '', + document: '' + document + '', } return parsePreparedContent( url, @@ -337,7 +337,7 @@ export const parsePreparedContent = async ( logger.info('parse-article completed') return { - domContent: preparedDocument.document, + domContent: document, parsedContent: article, canonicalUrl, pageType: parseOriginalContent(dom), diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index fa25b4732..eb61cac93 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -409,32 +409,6 @@ function getUrl(req) { return parsed.href; } -async function blockResources(client) { - const blockedResources = [ - // Assets - // '*/favicon.ico', - // '.css', - // '.jpg', - // '.jpeg', - // '.png', - // '.svg', - // '.woff', - - // Analytics and other fluff - '*.optimizely.com', - 'everesttech.net', - 'userzoom.com', - 'doubleclick.net', - 'googleadservices.com', - 'adservice.google.com/*', - 'connect.facebook.com', - 'connect.facebook.net', - 'sp.analytics.yahoo.com', - ] - - await client.send('Network.setBlockedURLs', { urls: blockedResources }); -} - async function retrievePage(url, logRecord, functionStartTime) { validateUrlString(url); @@ -494,8 +468,6 @@ async function retrievePage(url, logRecord, functionStartTime) { } catch {} }); - await blockResources(client); - /* * Disallow MathJax from running in Puppeteer and modifying the document, * we shall instead run it in our frontend application to transform any @@ -504,7 +476,8 @@ async function retrievePage(url, logRecord, functionStartTime) { await page.setRequestInterception(true); let requestCount = 0; page.on('request', request => { - if (['font', 'image', 'media'].includes(request.resourceType())) { + if (request.resourceType() === 'font') { + // Disallow fonts from loading request.abort(); return; } From 69486a8527664f809bb77423e3304d41b0785d7b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 13 Feb 2023 17:01:48 +0800 Subject: [PATCH 3/7] Add readability test for yuyue.com --- .../yuyue.com/expected-metadata.json | 12 + .../test/test-pages/yuyue.com/expected.html | 154 + .../test/test-pages/yuyue.com/source.html | 34886 ++++++++++++++++ .../test/test-pages/yuyue.com/url.txt | 1 + 4 files changed, 35053 insertions(+) create mode 100644 packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json create mode 100644 packages/readabilityjs/test/test-pages/yuyue.com/expected.html create mode 100644 packages/readabilityjs/test/test-pages/yuyue.com/source.html create mode 100644 packages/readabilityjs/test/test-pages/yuyue.com/url.txt diff --git a/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json new file mode 100644 index 000000000..4a1772b83 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "读《如何高效学习》—Madlife · 语雀", + "byline": null, + "dir": null, + "excerpt": "一句话，用整体性学习，即把新旧知识链起来成网，而不是分...", + "siteName": null, + "siteIcon": "https://mdn.alipayobjects.com/huamei_0prmtq/afts/img/A*sRUdR543RjcAAAAAAAAAAAAADvuFAQ/original", + "previewImage": "https://cdn.nlark.com/yuque/0/2022/png/22724648/1671339142303-ce7c7caa-57b6-473b-8fb3-bfaf59a79d3b.png", + "publishedDate": null, + "language": "English", + "readerable": false +} diff --git a/packages/readabilityjs/test/test-pages/yuyue.com/expected.html b/packages/readabilityjs/test/test-pages/yuyue.com/expected.html new file mode 100644 index 000000000..b6b6225a8 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/yuyue.com/expected.html @@ -0,0 +1,154 @@ +
+
+
+

一句话，用整体性学习，即把新旧知识链起来成网，而不是分割成一块一块。

+

作者用了三个观点和五个步骤来介绍整体性学习，如下：

+

整体性学习基于三个观点：

+

1、结构
相当于一座座城市，语文是一座，数学是一座。发达的城市，学有关的东西就会很快，例如学表达，写作。聪明人就是把新东西和老城市建立交易，联系越多越容易学，条条大路通新知识。完成自觉建立知识间联系的习惯，很容易学好。也就是说，只要新知识能放进城市里，建立起来联系，那就很容易学。
已经成熟的结构有
●感知结构：眼、鼻、耳、舌、手。最基本的结构，也发展的最好。
●关系结构：各种关系，人人、人事等等。
●基础数学结构：代数、函数等，用来简化其他学科内的关系。

+

2、模型
属于简化的结构，是城市的缩略图，是书的目录。模型的形式不限制，主要是为了压缩信息，将核心概念整合在一起，是城市的地基，最核心的部分，在模型的基础上引申出全部的知识。模型可以是一张图，一句话。后续的新知识在模型的基础上进行联系和优化。模型可以改进和优化，比如用视觉化。

+

3、高速公路
城市之间的连接，例如正在学习生物，将其与熟悉的商业发展史建立联系。高速公路可以激发创造力，将常人眼里风马牛不相及的专业联系在一起，将知识变得更有弹性。

+

整体性学习的五个步骤

+

1、获取
①获取的信息要准确：信息来源尽可能质量高一点。得到、或者比较好的课程。
②信息要简化：废话删掉。用笔记流加适当图表。
（1）不要呆板、僵硬、貌似层次分明的笔记。（2）在写下来的观点之间建立联系。

+
+
+

STHY IN ONE PAGE

+

MESIN ISU'L CESIUNT . NORL REART

+

DMHRU

+

NHER QUARTILE DANGE

+

189

+

压缩笔记示例

+

(第-133

+

(一页纸概括了一门课的所有观点)

+

VERIANIS

+

SHEPIR SUAR SOARY

+

之一部

+

LBEEEFFJ

+

PGOLUC ING

+

AWUNTILE QUTY PERCOATAGE

+

HORMAT QUARHILE UIAT

+

RE NFENFEAY

+

MAOICTBINS!

+

VALUE& OF OF.

+

SARTPLE

+

VERS

+

YY RESPONSE

+

FIEANCNTY.9RAND

+

BLANPLANATORY

+

中国石化

+

CERRELATION A 二

+

MVHANASG

+

WIGHRION

+

EFP AINGT BY

+

H-!

+

QUNER,ENSS

+

THABE VED PONIRE

+

王云以

+

DESS ANE CENT OFROTPOWE CENE CEN

+

LE EST SQVARES GEGSI ON LINE

+

5.5

+

LBUT

+

19

+

TRAPOIPLEN

+
+

image.png +

+
+


③容量尽可能大：一年读2本和一年读100本的差别
④速度尽可能快：阅读方法要好，同时不能漏掉太多重点。推荐用指读法。

+

2、理解（停留在这一步就是死记硬背式学习）
①字面意思是什么，每个字母代表什么。
②怎么得来的，和上下文有联系吗。
③如果无法理解，就往更细的点拆分，直至理解。

+

3、拓展
①这才是整体性学习的真正开始，用已有的模型来简化新知识的结构。
②分三种方式

+
+
+

知识的背景探究:从何而来,有什么

+

深度拓展(最

+

试验,论据是什么

+

耗时)

+

类似的结论有哪些,异同点是什么,

+

横向拓展

+

同时期的发现有哪些,同领域的发现

+

有哪些

+

在结构间建立高速公路

+

(1)为你正在学习的东西创造脑海中的图像.

+

纵向拓展(最

+

难,最具创造

+

较好的方法是,比喻法和内在化

+

内在化即图像化

+

(2)在这幅图像上加入其他感官和情感.

+

性)

+

(3)寻找图像不适用或不足的地方,防止产生

+

错误的联系

+
+

image.png +

+
+

4、纠错
①发现一些特例并指出，删除一些现实中不存在的联系，删除错误结论。
②不断实践，搞清楚是概念本身的问题还是理解问题；每种类型都实践下；每天都练练，不要临时突击。

+

5、应用：创造途径，将所学应用到生活中去

+

0、测试：伴随每一步，主要是改进技术

+
+
+

获取:我以前看过或听过这个吗

+

理解:我理解知识的含义吗

+

拓展:我知道它从何而来,与哪些知识有联系吗

+

-纠错:我删了错误结论以及不恰当的联系吗

+

应用:我将知识应用到实际生活中了吗

+
+

image.png +

+
+

另外讲了信息的类型
1、随意信息：事实、日期、定义或规则，没逻辑和规律。可以用联想来处理（最弱的信息，最难进行整体性学习，尽可能赋予一点逻辑，在无序中找有序）
2、观点信息：存在争论的信息，用图表法来处理
3、过程信息：教你怎么行动的信息，如游泳，需要不断练习，可以用内在化、比喻法来改进。（强信息，可以直接唤起新旧知识的联系）
4、具体信息：信息和感官可以联系在一起（强信息，可以直接唤起新旧知识的联系）
5、抽象信息：有逻辑，但是没图像，非常抽象（看过《别闹了，费曼先生》，就知道费曼最过人之处就是别人看到的是抽象物理知识，而他看到的是实际生活）

信息处理的精髓即，将弱信息结构转为强信息结构。

+

讲了这样学习的好处
通过联想，所有的观点、知识都会变得有用，尝试将你的课程与感兴趣的东西联系在一起，任何所学知识都要发掘其实际的用处，特别是对于改进自身有什么实际用处。
（1）统计学──我利用统计学知识给本书起名字以及定价。利用谷歌搜索引擎，我尝试各种名字和价格，最终利用统计学决定了这个书名和价格最为吸引人。
（2）计算机──除了编程这种明显的应用之外，我还发现计算机科学是寻找问题的有用途径，纠错、算法都是可以借鉴到其他地方的有用思想。
（3）会计学──会计学能帮助我理清个人财务以及报税。运用基本的会计学原理整理我的个人财务，使它们看起来一目了然。
（4）经济学──经济学教会我重新看待金钱在社会中的价值，明白了金钱仅仅是物质交换的载体后，我的个人哲学体系也随之发生了很多变化。
（5）历史──历史是了解现在的工具，通过学习古代亚洲史能帮助我们看清现代中国、印度和日本的种种问题。历史就是照亮现在的一面镜子。

+

最后讲了如何提高效率
1、能量管理：关于能量管理，最好的一本书是《精力管理：管理精力，而非时间，是高效、健康与快乐的基础》（The Powerof Full Engagement）
2、保持整体性学习的闭环实践，而不是只做其中一部分。
3、不拖延，用周目标和日目标来分配工作

+
+
+

这是我的周/日(W/D)目标体系.

+

MY LISTS THLS LLST:EDIT I REORDER I SHARS

+

周目标:2月4~10日

+

每周博客更新

+

PTB文档

+

MY LISTS THIS LIST:EDIT RCORDEE I SHARE

+

"FLEX"

+

文档

+

日目标:2008年2月5日

+

网址备份

+

课程

+

阅读周四ENT案例

+

健身

+

阅读下一个周四ENT案例

+

演讲

+

土司马斯

+

(TOASTMASTERS)

+

我使用TADALIST,这是一个

+

阅读会计学第5章

+

在线程序.每天晚上我都要检查

+

阅读会计学第5章

+

自己的每日目标和每周目标完成

+

阅读周四ENT案例

+

情况,以确保最终实现目标.

+
+

image.png +

+
+


4、批量处理：《批处理：节省时间、减轻压力的20个小技巧》。
5、有序：某些物品放在固定位置；有随时记录想法的地方；坚持写清单和日历。
6、养成习惯并每天都坚持：《如何改变一个习惯》（How to Change a Habit）。


+
+
+

从智力挑战开始

+

智力挑战的目的是养成新的习惯,许多新方法一开始用起来都很费时间.练

+

习2~4周后,速度和效果会大大提高.最后,你需要根据学习的目标对方法进行适当

+

的改进.

+

以下是一些重要的技巧.

+

(1)至少坚持了周.可能你想学会本书介绍的很多技术,但是要记住,如果你不

+

坚持3周以上的智力挑战练习,很难将新技术变成一种习惯.

+

(2)一次只做一个.不要试图一次完成几个智力挑战,一次只专注一个.

+

(3)比喻,内在化和图表法优先.这些是整体性学习中的核心技术,就从它们

+

开始吧!

+

(4)使用奖励材料.本书附赠有6段专门设计用于练习智力挑战的奖励材料,利

+

用它们会让你更轻松些.

+

(5)记录下学习的过程.练习智力挑战时,坚持写一句话日记,用一两句话记

+

下你的体会和感受,有助于你坚持下来和解决练习中的各种问题.

+
+

image.png +

+
+
+

​

+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/yuyue.com/source.html b/packages/readabilityjs/test/test-pages/yuyue.com/source.html new file mode 100644 index 000000000..6815fdd85 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/yuyue.com/source.html @@ -0,0 +1,34886 @@ + + + + + + + + + + + + + + + + + + + + + 读《如何高效学习》—Madlife + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+
+
+
+
+
+
+

+ 读《如何高效学习》—Madlife +

+
+
+
+
+
+
+
+
+
+
+
+ 一句话，用整体性学习，即把新旧知识链起来成网，而不是分割成一块一块。
 +
+
+
作者用了三个观点和五个步骤来介绍整体性学习，如下：
 +
+
+
整体性学习基于三个观点：
 +
+
+
1、结构
相当于一座座城市，语文是一座，数学是一座。发达的城市，学有关的东西就会很快，例如学表达，写作。聪明人就是把新东西和老城市建立交易，联系越多越容易学，条条大路通新知识。完成自觉建立知识间联系的习惯，很容易学好。也就是说，只要新知识能放进城市里，建立起来联系，那就很容易学。
已经成熟的结构有
●感知结构：眼、鼻、耳、舌、手。最基本的结构，也发展的最好。
●关系结构：各种关系，人人、人事等等。
●基础数学结构：代数、函数等，用来简化其他学科内的关系。
 +
+
+
2、模型
属于简化的结构，是城市的缩略图，是书的目录。模型的形式不限制，主要是为了压缩信息，将核心概念整合在一起，是城市的地基，最核心的部分，在模型的基础上引申出全部的知识。模型可以是一张图，一句话。后续的新知识在模型的基础上进行联系和优化。模型可以改进和优化，比如用视觉化。
 +
+
+
3、高速公路
城市之间的连接，例如正在学习生物，将其与熟悉的商业发展史建立联系。高速公路可以激发创造力，将常人眼里风马牛不相及的专业联系在一起，将知识变得更有弹性。
 +
+
+
整体性学习的五个步骤
 +
+
+
1、获取
①获取的信息要准确：信息来源尽可能质量高一点。得到、或者比较好的课程。
②信息要简化：废话删掉。用笔记流加适当图表。
（1）不要呆板、僵硬、貌似层次分明的笔记。（2）在写下来的观点之间建立联系。
 +
+
+
+
+ +
+
+
+
+ STHY IN ONE PAGE +
+
+ MESIN ISU'L CESIUNT . NORL REART +
+
+ DMHRU +
+
+ NHER QUARTILE DANGE +
+
+ 189 +
+
+ 压缩笔记示例 +
+
+ (第-133 +
+
+ (一页纸概括了一门课的所有观点) +
+
+ VERIANIS +
+
+ SHEPIR SUAR SOARY +
+
+ 之一部 +
+
+ LBEEEFFJ +
+
+ PGOLUC ING +
+
+ AWUNTILE QUTY PERCOATAGE +
+
+ HORMAT QUARHILE UIAT +
+
+ RE NFENFEAY +
+
+ MAOICTBINS! +
+
+ VALUE& OF OF. +
+
+ SARTPLE +
+
+ VERS +
+
+ YY RESPONSE +
+
+ FIEANCNTY.9RAND +
+
+ BLANPLANATORY +
+
+ 中国石化 +
+
+ CERRELATION A 二 +
+
+ MVHANASG +
+
+ WIGHRION +
+
+ EFP AINGT BY +
+
+ H-! +
+
+ QUNER,ENSS +
+
+ THABE VED PONIRE +
+
+ 王云以 +
+
+ DESS ANE CENT OFROTPOWE CENE CEN +
+
+ LE EST SQVARES GEGSI ON LINE +
+
+ 5.5 +
+
+ LBUT +
+
+ 19 +
+
+ TRAPOIPLEN +
+
+
+ image.png +
+
+

③容量尽可能大：一年读2本和一年读100本的差别
④速度尽可能快：阅读方法要好，同时不能漏掉太多重点。推荐用指读法。
 +
+
+
2、理解（停留在这一步就是死记硬背式学习）
①字面意思是什么，每个字母代表什么。
②怎么得来的，和上下文有联系吗。
③如果无法理解，就往更细的点拆分，直至理解。
 +
+
+
3、拓展
①这才是整体性学习的真正开始，用已有的模型来简化新知识的结构。
②分三种方式
 +
+
+
+
+ +
+
+
+
+ 知识的背景探究:从何而来,有什么 +
+
+ 深度拓展(最 +
+
+ 试验,论据是什么 +
+
+ 耗时) +
+
+ 类似的结论有哪些,异同点是什么, +
+
+ 横向拓展 +
+
+ 同时期的发现有哪些,同领域的发现 +
+
+ 有哪些 +
+
+ 在结构间建立高速公路 +
+
+ (1)为你正在学习的东西创造脑海中的图像. +
+
+ 纵向拓展(最 +
+
+ 难,最具创造 +
+
+ 较好的方法是,比喻法和内在化 +
+
+ 内在化即图像化 +
+
+ (2)在这幅图像上加入其他感官和情感. +
+
+ 性) +
+
+ (3)寻找图像不适用或不足的地方,防止产生 +
+
+ 错误的联系 +
+
+
+ image.png +
+
+

+
+
+
4、纠错
①发现一些特例并指出，删除一些现实中不存在的联系，删除错误结论。
②不断实践，搞清楚是概念本身的问题还是理解问题；每种类型都实践下；每天都练练，不要临时突击。
 +
+
+
5、应用：创造途径，将所学应用到生活中去
 +
+
+
0、测试：伴随每一步，主要是改进技术
 +
+
+
+
+ +
+
+
+
+ 获取:我以前看过或听过这个吗 +
+
+ 理解:我理解知识的含义吗 +
+
+ 拓展:我知道它从何而来,与哪些知识有联系吗 +
+
+ -纠错:我删了错误结论以及不恰当的联系吗 +
+
+ 应用:我将知识应用到实际生活中了吗 +
+
+
+ image.png +
+
+

+
+
+
另外讲了信息的类型
1、随意信息：事实、日期、定义或规则，没逻辑和规律。可以用联想来处理（最弱的信息，最难进行整体性学习，尽可能赋予一点逻辑，在无序中找有序）
2、观点信息：存在争论的信息，用图表法来处理
3、过程信息：教你怎么行动的信息，如游泳，需要不断练习，可以用内在化、比喻法来改进。（强信息，可以直接唤起新旧知识的联系）
4、具体信息：信息和感官可以联系在一起（强信息，可以直接唤起新旧知识的联系）
5、抽象信息：有逻辑，但是没图像，非常抽象（看过《别闹了，费曼先生》，就知道费曼最过人之处就是别人看到的是抽象物理知识，而他看到的是实际生活）

信息处理的精髓即，将弱信息结构转为强信息结构。
 +
+
+
讲了这样学习的好处
通过联想，所有的观点、知识都会变得有用，尝试将你的课程与感兴趣的东西联系在一起，任何所学知识都要发掘其实际的用处，特别是对于改进自身有什么实际用处。
（1）统计学──我利用统计学知识给本书起名字以及定价。利用谷歌搜索引擎，我尝试各种名字和价格，最终利用统计学决定了这个书名和价格最为吸引人。
（2）计算机──除了编程这种明显的应用之外，我还发现计算机科学是寻找问题的有用途径，纠错、算法都是可以借鉴到其他地方的有用思想。
（3）会计学──会计学能帮助我理清个人财务以及报税。运用基本的会计学原理整理我的个人财务，使它们看起来一目了然。
（4）经济学──经济学教会我重新看待金钱在社会中的价值，明白了金钱仅仅是物质交换的载体后，我的个人哲学体系也随之发生了很多变化。
（5）历史──历史是了解现在的工具，通过学习古代亚洲史能帮助我们看清现代中国、印度和日本的种种问题。历史就是照亮现在的一面镜子。
 +
+
+
最后讲了如何提高效率
1、能量管理：关于能量管理，最好的一本书是《精力管理：管理精力，而非时间，是高效、健康与快乐的基础》（The Powerof Full Engagement）
2、保持整体性学习的闭环实践，而不是只做其中一部分。
3、不拖延，用周目标和日目标来分配工作
 +
+
+
+
+ +
+
+
+
+ 这是我的周/日(W/D)目标体系. +
+
+ MY LISTS THLS LLST:EDIT I REORDER I SHARS +
+
+ 周目标:2月4~10日 +
+
+ 每周博客更新 +
+
+ PTB文档 +
+
+ MY LISTS THIS LIST:EDIT RCORDEE I SHARE +
+
+ "FLEX" +
+
+ 文档 +
+
+ 日目标:2008年2月5日 +
+
+ 网址备份 +
+
+ 课程 +
+
+ 阅读周四ENT案例 +
+
+ 健身 +
+
+ 阅读下一个周四ENT案例 +
+
+ 演讲 +
+
+ 土司马斯 +
+
+ (TOASTMASTERS) +
+
+ 我使用TADALIST,这是一个 +
+
+ 阅读会计学第5章 +
+
+ 在线程序.每天晚上我都要检查 +
+
+ 阅读会计学第5章 +
+
+ 自己的每日目标和每周目标完成 +
+
+ 阅读周四ENT案例 +
+
+ 情况,以确保最终实现目标. +
+
+
+ image.png +
+
+

4、批量处理：《批处理：节省时间、减轻压力的20个小技巧》。
5、有序：某些物品放在固定位置；有随时记录想法的地方；坚持写清单和日历。
6、养成习惯并每天都坚持：《如何改变一个习惯》（How to Change a Habit）。


 +
+
+
+
+ +
+
+
+
+ 从智力挑战开始 +
+
+ 智力挑战的目的是养成新的习惯,许多新方法一开始用起来都很费时间.练 +
+
+ 习2~4周后,速度和效果会大大提高.最后,你需要根据学习的目标对方法进行适当 +
+
+ 的改进. +
+
+ 以下是一些重要的技巧. +
+
+ (1)至少坚持了周.可能你想学会本书介绍的很多技术,但是要记住,如果你不 +
+
+ 坚持3周以上的智力挑战练习,很难将新技术变成一种习惯. +
+
+ (2)一次只做一个.不要试图一次完成几个智力挑战,一次只专注一个. +
+
+ (3)比喻,内在化和图表法优先.这些是整体性学习中的核心技术,就从它们 +
+
+ 开始吧! +
+
+ (4)使用奖励材料.本书附赠有6段专门设计用于练习智力挑战的奖励材料,利 +
+
+ 用它们会让你更轻松些. +
+
+ (5)记录下学习的过程.练习智力挑战时,坚持写一句话日记,用一两句话记 +
+
+ 下你的体会和感受,有助于你坚持下来和解决练习中的各种问题. +
+
+
+ image.png +
+
+

+
+
+
+
+
+
+
+ ​ +
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+ Madlife +
+
+ 01-20 02:55 +
+
+ 126 +
+
+ 0 +
+
+ IP region浙江 +
+
+ Report +
+
+
+
+ +
+
+
+
+
+
+
+
+ Markup comments (0) +
+
+ + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Sign Up / Login Yuque to comment +
+
+
+
+
+
+ + +
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + Madlife +
+
+
+
+ + + + + +
+
+ 读书笔记 +
+ + + + + +
+
+
+
+
+ +
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ + +
+
+
+
+ + + + + + + + + + + + + + + + + + + + +
+ +
+ + diff --git a/packages/readabilityjs/test/test-pages/yuyue.com/url.txt b/packages/readabilityjs/test/test-pages/yuyue.com/url.txt new file mode 100644 index 000000000..875f4e29a --- /dev/null +++ b/packages/readabilityjs/test/test-pages/yuyue.com/url.txt @@ -0,0 +1 @@ +https://www.yuque.com/u22288095/gf5dgh/yoqgzsrsdltgeel9 \ No newline at end of file From 69b87078a5a3a89ac5dd9d9b96fd7624fc351f37 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 13 Feb 2023 22:10:56 +0800 Subject: [PATCH 4/7] Fix not timeout if scroll more than 5 seconds --- packages/puppeteer-parse/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index eb61cac93..a9da97977 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -7,7 +7,6 @@ const { encode } = require("urlsafe-base64"); const crypto = require("crypto"); const Url = require('url'); -// const puppeteer = require('puppeteer-extra'); const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); @@ -26,9 +25,10 @@ puppeteer.use(StealthPlugin()); // Add adblocker plugin to block all ads and trackers (saves bandwidth) const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); -const createDOMPurify = require("dompurify"); puppeteer.use(AdblockerPlugin({ blockTrackers: true })); +const createDOMPurify = require("dompurify"); + const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined; @@ -296,7 +296,7 @@ async function fetchContent(req, res) { if (contentType === 'application/pdf') { const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); - const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); + await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); } else { if (!content || !title) { const result = await retrieveHtml(page, logRecord); @@ -557,7 +557,7 @@ async function retrieveHtml(page, logRecord) { } })(); }), - await page.waitForTimeout(5000), + page.waitForTimeout(5000), ]); logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; From 4513b5931bb82d2d25e77115ee97057c7b1c8091 Mon Sep 17 00:00:00 2001 From: sywhb Date: Mon, 13 Feb 2023 14:14:39 +0000 Subject: [PATCH 5/7] Update generated html --- packages/readabilityjs/test/index.html | 12 + .../test/test-pages/habr.com/distiller.html | 243 ++++++++++++++++++ .../test/test-pages/yuyue.com/distiller.html | 220 ++++++++++++++++ 3 files changed, 475 insertions(+) create mode 100644 packages/readabilityjs/test/test-pages/habr.com/distiller.html create mode 100644 packages/readabilityjs/test/test-pages/yuyue.com/distiller.html diff --git a/packages/readabilityjs/test/index.html b/packages/readabilityjs/test/index.html index c4f1ec163..a6b272a3f 100644 --- a/packages/readabilityjs/test/index.html +++ b/packages/readabilityjs/test/index.html @@ -266,6 +266,12 @@ [dom-distiller] +
  • habr.com
    + [source] + [readability] + [dom-distiller] +
    • +
  • fast-company
    [source] [readability] @@ -416,6 +422,12 @@ [dom-distiller]
    • +
  • yuyue.com
    + [source] + [readability] + [dom-distiller] +
    • +
  • debugger.medium
    [source] [readability] diff --git a/packages/readabilityjs/test/test-pages/habr.com/distiller.html b/packages/readabilityjs/test/test-pages/habr.com/distiller.html new file mode 100644 index 000000000..c1506e446 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/habr.com/distiller.html @@ -0,0 +1,243 @@ +
    + 4 мин +
    22K +
    Картинка, конечно, стронгли анрилейтед
    +
    + Разные трюки я тестировал на Google Chrome 107.0.5304.107 и Mozilla Firefox 107.0 на Windows 10.
    +
    + Чтобы результаты всегда были железно воспроизводимыми, я отключил все С-State’ы, ядра зафиксировал на 5 ГГц.
    +
    + У меня 9900К, это Coffee Lake c AVX256, какие оптимизации применит Jit для вашего процессора — я не знаю, результат на вашем компьютере может отличаться от моего, в т.ч. из-за микроархитектуры процессора.
    +
    + Скорость парсинга кода тоже входит в бенчмарк, поэтому браузер с быстрым парсером будет впереди.
    +
    +

    + Есть ли у переменной оверхед? +


    + Есть ли смысл использовать только dot notation? Какова цена выноса лишней переменной?
    +
    + 
    var array = new Array(65535).fill()
+
+
+var a = array.map((x) => x)
+var b = array.map((x) => x)
+var c = array.map((x) => x)
+
+
+var a = array.map((x) => x)
+var b = array.map((x) => x).map((x) => x)
+
+
+var a = array.map((x) => x).map((x) => x).map((x) => x)

    + Чтобы узнать, гоняет ли браузер память туда-сюда, делаем мы .map на массив длиной 65535 с нулями внутри. Линк.
    +
    + Здесь и далее в качестве единицы измерения на графиках будет указано кол-во выполнений кода в секунду, включая парсинг и компиляцию. Больше — лучше
    +
    + Хром не заметил разницы, а вот лиса заметила. Применительно к лисе, у лишней переменной есть измеряемый оверхед.
    +
    +

    + Есть ли разница между var, let, const или их отсутствием? +


    +
    + Проверим. Используя разные биндинги — создадим POJO с переменной е. Потом добавим ему функцию о и запустим эту функцию. Бенчмарк простой, но движущихся частей много. Линк.
    +
    + 
    var g = { e: [] }
+g.o = function(x) { g.e.push(...[1,2,3]) }
+g.o()

    + Код выглядит так, отличаются только биндинги.
    +
    +
    + Результат неожиданный, но железно воспроизводимый. var, быстрее.
    +
    +

    + Bounce pattern, Switch case, длинная тернарка +


    + Если обе конструкции логически одинаковые, они должны строить одно и то же синтаксическое дерево, верно? Давайте проверим.
    +
    + 
    
+function thing(e) {
+    switch (e) {
+      case 0:
+        return "0";
+
+      case 1:
+        return "1";
+
+      case 2:
+        return "2";
+
+      case 3:
+        return "3";
+        
+      default:
+        return "";
+    }
+}
+
+
+function bounce(x)
+{
+   if (x === 0) return "0";
+   if (x === 1) return "1";
+   if (x === 2) return "2";
+   if (x === 3) return "3";
+   
+   return ""
+}
+
+
+function bounce(x) {
+  return 0 === x ? "0" : 1 === x ? "1" : 2 === x ? "2" : 3 === x ? "3" : "";
+}

    + Вот так выглядит код. Для всех вариантов он одинаков, отличаются только вызовы.
    +
    +

    + ▍ 1. Вызов в цикле + 

    for (let t = 0; 1e5 > t; t++) bounce(0), bounce(2), bounce(6);

    + Вызов выглядит так. Линк.
    +
    +
    +

    + ▍ 2. В цикле с другим типом + 

    for (let t = 0; 1e5 > t; t++) bounce("0"), bounce("2"), bounce("");

    + Тут мы покидываем строку вместо числа. В свитче и if блоках используется строгое равенство, поэтому свитч выходит только через default, а if’ы выходят только последний return. Линк.
    +
    +
    +

    + ▍ 3. Без цикла + 

    bounce(0), bounce(2), bounce(6)

    + Просто три вызова подряд, никаких циклов. Линк.
    +
    +
    + Похоже, что после первоначальной компиляции лиса не пытается дальше оптимизировать цикл, как это делает хром.
    +
    + Также лиса, похоже, не строит одно и то же AST, как это делает хром. Рекомендую заменить ваши длинные if’ы и bounce паттерны на свитчи, чтобы избежать лисиных тормозов.
    +
    +

    + Инициализация массива +


    + Для примера возьму из паттернов функционального программирования, когда ты инициализируешь массив, прокидывая лямбду в инициализатор. Просто ради примера, в качестве этой лямбды будет fizzbuzz.
    +
    + 
    var times = 65535;
+
+function initializer(val, z) {
+    const i = z % 5 | 0;
+    return 0 == (z % 3 | 0) ? 0 === i ? "fizzbuzz" : "fizz" : 0 === i ? "buzz" : z;
+}
+
+
+var b = new Array(times);
+for (var i = 0; i < times; i++) {
+    b[i] = initializer(b[i], i)
+}
+b
+
+
+var c = [];
+for (var i = 0; i < times; i++) {
+    c.push(initializer(c[i], i))
+}
+c
+
+
+new Array(times).fill().map(initializer)
+

    + Это не самый красивый fizzbuzz, но это мой fizzbuzz. Линк на бенч.
    +
    +
    + Вариант с fill map создаёт два массива, сначала при вызове конструктора, потом при вызове map. Но такой вариант безальтернативно быстрее на хроме.
    +
    +

    + Конкатенация массивов + 

    
+arr.reduce((acc, val) => acc.concat(val), [])
+
+
+arr.flatMap(x => x)
+
+
+arr.flat()
+
+
+arr.reduce((acc, val) => {
+    if (val) val.forEach(a => acc.push(a));
+    return acc;
+}, [])
+
+
+let acc = [];
+
+arr.forEach(val => {
+    val && val.forEach(v => acc.push(v));
+}), acc;
+
+
+[].concat(...arr)
+

    + Конкатенация массивов на 1 уровень, поведение идентичное flat(1). Линк.
    +
    +
    + Иногда я не понимаю, почему разработчики движков оставили такой потенциал для оптимизации.
    +
    +

    + Уничтожение хрома +


    + Бенчмарки ниже я перепроверял по нескольку раз, результат одинаковый и верный. Лиса действительно такая быстрая.
    +
    +

    + ▍ Итерация по массиву +


    + Сравнивать будем Array.prototype.forEach vs for...of vs for. На код смотрите по линку.
    +
    +
    + Ради производительности, циклы for, лучше переделать в forEach, чтобы хром не отставал.
    +
    +

    + ▍ Содержит ли строка значение + 

    
+url.includes('matchthis')
+
+
+/matchthis/.test(url)
+
+
+url.match(/matchthis/).length >= 0
+
+
+url.indexOf('matchthis') >= 0
+
+
+url.search('matchthis') >= 0
+

    +
    +
    + Трюк с IndexOf быстрее и на лисе, и на хроме. Используйте трюк с IndexOf. Линк на бенчмарк.
    +
    +

    + Преобразование строки в число +


    + Тестируем неявное преобразование, парсинг и вызов конструктора.
    +
    + 
    
+var imp = + strNum
+
+
+var toStr = parseFloat(strNum)
+
+
+var num = Number(strNum)
+

    + ▍ Float +


    +
    +
    + Я перепроверял, это не ошибка. Неявный каст стринги в инт практически бесплатный у лисы. Линк на бенч.
    +
    +

    + Выводы +

    1. Лисичка похорошела.
    2. JS сделан за неделю на коленке.
    3. Я не пишу на JS.
    4. Вы тоже прекращайте.
    \ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/yuyue.com/distiller.html b/packages/readabilityjs/test/test-pages/yuyue.com/distiller.html new file mode 100644 index 000000000..6ca990185 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/yuyue.com/distiller.html @@ -0,0 +1,220 @@ +
    + 一句话，用整体性学习，即把新旧知识链起来成网，而不是分割成一块一块。
     +
    作者用了三个观点和五个步骤来介绍整体性学习，如下：
     +
    整体性学习基于三个观点：
     +
    1、结构
    相当于一座座城市，语文是一座，数学是一座。发达的城市，学有关的东西就会很快，例如学表达，写作。聪明人就是把新东西和老城市建立交易，联系越多越容易学，条条大路通新知识。完成自觉建立知识间联系的习惯，很容易学好。也就是说，只要新知识能放进城市里，建立起来联系，那就很容易学。
    已经成熟的结构有
    ●感知结构：眼、鼻、耳、舌、手。最基本的结构，也发展的最好。
    ●关系结构：各种关系，人人、人事等等。
    ●基础数学结构：代数、函数等，用来简化其他学科内的关系。
     +
    2、模型
    属于简化的结构，是城市的缩略图，是书的目录。模型的形式不限制，主要是为了压缩信息，将核心概念整合在一起，是城市的地基，最核心的部分，在模型的基础上引申出全部的知识。模型可以是一张图，一句话。后续的新知识在模型的基础上进行联系和优化。模型可以改进和优化，比如用视觉化。
     +
    3、高速公路
    城市之间的连接，例如正在学习生物，将其与熟悉的商业发展史建立联系。高速公路可以激发创造力，将常人眼里风马牛不相及的专业联系在一起，将知识变得更有弹性。
     +
    整体性学习的五个步骤
     +
    1、获取
    ①获取的信息要准确：信息来源尽可能质量高一点。得到、或者比较好的课程。
    ②信息要简化：废话删掉。用笔记流加适当图表。
    （1）不要呆板、僵硬、貌似层次分明的笔记。（2）在写下来的观点之间建立联系。
     +
    + STHY IN ONE PAGE +
    + MESIN ISU'L CESIUNT . NORL REART +
    + DMHRU +
    + NHER QUARTILE DANGE +
    + 189 +
    + 压缩笔记示例 +
    + (第-133 +
    + (一页纸概括了一门课的所有观点) +
    + VERIANIS +
    + SHEPIR SUAR SOARY +
    + 之一部 +
    + LBEEEFFJ +
    + PGOLUC ING +
    + AWUNTILE QUTY PERCOATAGE +
    + HORMAT QUARHILE UIAT +
    + RE NFENFEAY +
    + MAOICTBINS! +
    + VALUE& OF OF. +
    + SARTPLE +
    + VERS +
    + YY RESPONSE +
    + FIEANCNTY.9RAND +
    + BLANPLANATORY +
    + 中国石化 +
    + CERRELATION A 二 +
    + MVHANASG +
    + WIGHRION +
    + EFP AINGT BY +
    + H-! +
    + QUNER,ENSS +
    + THABE VED PONIRE +
    + 王云以 +
    + DESS ANE CENT OFROTPOWE CENE CEN +
    + LE EST SQVARES GEGSI ON LINE +
    + 5.5 +
    + LBUT +
    + 19 +
    + TRAPOIPLEN +
    image.png

    ③容量尽可能大：一年读2本和一年读100本的差别
    ④速度尽可能快：阅读方法要好，同时不能漏掉太多重点。推荐用指读法。
     +
    2、理解（停留在这一步就是死记硬背式学习）
    ①字面意思是什么，每个字母代表什么。
    ②怎么得来的，和上下文有联系吗。
    ③如果无法理解，就往更细的点拆分，直至理解。
     +
    3、拓展
    ①这才是整体性学习的真正开始，用已有的模型来简化新知识的结构。
    ②分三种方式
     +
    + 知识的背景探究:从何而来,有什么 +
    + 深度拓展(最 +
    + 试验,论据是什么 +
    + 耗时) +
    + 类似的结论有哪些,异同点是什么, +
    + 横向拓展 +
    + 同时期的发现有哪些,同领域的发现 +
    + 有哪些 +
    + 在结构间建立高速公路 +
    + (1)为你正在学习的东西创造脑海中的图像. +
    + 纵向拓展(最 +
    + 难,最具创造 +
    + 较好的方法是,比喻法和内在化 +
    + 内在化即图像化 +
    + (2)在这幅图像上加入其他感官和情感. +
    + 性) +
    + (3)寻找图像不适用或不足的地方,防止产生 +
    + 错误的联系 +
    image.png
    4、纠错
    ①发现一些特例并指出，删除一些现实中不存在的联系，删除错误结论。
    ②不断实践，搞清楚是概念本身的问题还是理解问题；每种类型都实践下；每天都练练，不要临时突击。
     +
    5、应用：创造途径，将所学应用到生活中去
     +
    0、测试：伴随每一步，主要是改进技术
     +
    + 获取:我以前看过或听过这个吗 +
    + 理解:我理解知识的含义吗 +
    + 拓展:我知道它从何而来,与哪些知识有联系吗 +
    + -纠错:我删了错误结论以及不恰当的联系吗 +
    + 应用:我将知识应用到实际生活中了吗 +
    image.png
    另外讲了信息的类型
    1、随意信息：事实、日期、定义或规则，没逻辑和规律。可以用联想来处理（最弱的信息，最难进行整体性学习，尽可能赋予一点逻辑，在无序中找有序）
    2、观点信息：存在争论的信息，用图表法来处理
    3、过程信息：教你怎么行动的信息，如游泳，需要不断练习，可以用内在化、比喻法来改进。（强信息，可以直接唤起新旧知识的联系）
    4、具体信息：信息和感官可以联系在一起（强信息，可以直接唤起新旧知识的联系）
    5、抽象信息：有逻辑，但是没图像，非常抽象（看过《别闹了，费曼先生》，就知道费曼最过人之处就是别人看到的是抽象物理知识，而他看到的是实际生活）

    信息处理的精髓即，将弱信息结构转为强信息结构。
     +
    讲了这样学习的好处
    通过联想，所有的观点、知识都会变得有用，尝试将你的课程与感兴趣的东西联系在一起，任何所学知识都要发掘其实际的用处，特别是对于改进自身有什么实际用处。
    （1）统计学──我利用统计学知识给本书起名字以及定价。利用谷歌搜索引擎，我尝试各种名字和价格，最终利用统计学决定了这个书名和价格最为吸引人。
    （2）计算机──除了编程这种明显的应用之外，我还发现计算机科学是寻找问题的有用途径，纠错、算法都是可以借鉴到其他地方的有用思想。
    （3）会计学──会计学能帮助我理清个人财务以及报税。运用基本的会计学原理整理我的个人财务，使它们看起来一目了然。
    （4）经济学──经济学教会我重新看待金钱在社会中的价值，明白了金钱仅仅是物质交换的载体后，我的个人哲学体系也随之发生了很多变化。
    （5）历史──历史是了解现在的工具，通过学习古代亚洲史能帮助我们看清现代中国、印度和日本的种种问题。历史就是照亮现在的一面镜子。
     +
    最后讲了如何提高效率
    1、能量管理：关于能量管理，最好的一本书是《精力管理：管理精力，而非时间，是高效、健康与快乐的基础》（The Powerof Full Engagement）
    2、保持整体性学习的闭环实践，而不是只做其中一部分。
    3、不拖延，用周目标和日目标来分配工作
     +
    + 这是我的周/日(W/D)目标体系. +
    + MY LISTS THLS LLST:EDIT I REORDER I SHARS +
    + 周目标:2月4~10日 +
    + 每周博客更新 +
    + PTB文档 +
    + MY LISTS THIS LIST:EDIT RCORDEE I SHARE +
    + "FLEX" +
    + 文档 +
    + 日目标:2008年2月5日 +
    + 网址备份 +
    + 课程 +
    + 阅读周四ENT案例 +
    + 健身 +
    + 阅读下一个周四ENT案例 +
    + 演讲 +
    + 土司马斯 +
    + (TOASTMASTERS) +
    + 我使用TADALIST,这是一个 +
    + 阅读会计学第5章 +
    + 在线程序.每天晚上我都要检查 +
    + 阅读会计学第5章 +
    + 自己的每日目标和每周目标完成 +
    + 阅读周四ENT案例 +
    + 情况,以确保最终实现目标. +
    image.png

    4、批量处理：《批处理：节省时间、减轻压力的20个小技巧》。
    5、有序：某些物品放在固定位置；有随时记录想法的地方；坚持写清单和日历。
    6、养成习惯并每天都坚持：《如何改变一个习惯》（How to Change a Habit）。


     +
    + 从智力挑战开始 +
    + 智力挑战的目的是养成新的习惯,许多新方法一开始用起来都很费时间.练 +
    + 习2~4周后,速度和效果会大大提高.最后,你需要根据学习的目标对方法进行适当 +
    + 的改进. +
    + 以下是一些重要的技巧. +
    + (1)至少坚持了周.可能你想学会本书介绍的很多技术,但是要记住,如果你不 +
    + 坚持3周以上的智力挑战练习,很难将新技术变成一种习惯. +
    + (2)一次只做一个.不要试图一次完成几个智力挑战,一次只专注一个. +
    + (3)比喻,内在化和图表法优先.这些是整体性学习中的核心技术,就从它们 +
    + 开始吧! +
    + (4)使用奖励材料.本书附赠有6段专门设计用于练习智力挑战的奖励材料,利 +
    + 用它们会让你更轻松些. +
    + (5)记录下学习的过程.练习智力挑战时,坚持写一句话日记,用一两句话记 +
    + 下你的体会和感受,有助于你坚持下来和解决练习中的各种问题. +
    image.png
    + ​ +
    \ No newline at end of file From cc8b1cefdb654c9fc92d5cee49eb00649fc94af4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 14 Feb 2023 12:33:59 +0800 Subject: [PATCH 6/7] Preserve 
     elements with prism- class and identity
 them as code blocks

---
 packages/api/src/utils/parser.ts              |    4 +-
 packages/content-handler/src/index.ts         |    4 +-
 packages/readabilityjs/Readability.js         |    2 +-
 .../robinwieruch.de/expected-metadata.json    |   12 +
 .../test-pages/robinwieruch.de/expected.html  | 1469 ++++++++++
 .../test-pages/robinwieruch.de/source.html    | 2504 +++++++++++++++++
 .../test/test-pages/robinwieruch.de/url.txt   |    1 +
 7 files changed, 3993 insertions(+), 3 deletions(-)
 create mode 100644 packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json
 create mode 100644 packages/readabilityjs/test/test-pages/robinwieruch.de/expected.html
 create mode 100644 packages/readabilityjs/test/test-pages/robinwieruch.de/source.html
 create mode 100644 packages/readabilityjs/test/test-pages/robinwieruch.de/url.txt

diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts
index b1d8a1e8a..7fe72ebfd 100644
--- a/packages/api/src/utils/parser.ts
+++ b/packages/api/src/utils/parser.ts
@@ -239,7 +239,9 @@ export const parsePreparedContent = async (
     // to the handlers, and have some concept of postHandle
     if (article?.content) {
       const articleDom = parseHTML(article.content).document
-      const codeBlocks = articleDom.querySelectorAll('code')
+      const codeBlocks = articleDom.querySelectorAll(
+        'code, pre[class^="prism-"], pre[class^="language-"]'
+      )
       if (codeBlocks.length > 0) {
         codeBlocks.forEach((e) => {
           if (e.textContent) {
diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index fed3fbeab..230a3dd12 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -35,7 +35,7 @@ import { EveryIoHandler } from './newsletters/every-io-handler'
 import { EnergyWorldHandler } from './newsletters/energy-world'
 import { IndiaTimesHandler } from './newsletters/india-times-handler'
 
-const validateUrlString = (url: string) => {
+const validateUrlString = (url: string): boolean => {
   const u = new URL(url)
   // Make sure the URL is http or https
   if (u.protocol !== 'http:' && u.protocol !== 'https:') {
@@ -49,6 +49,8 @@ const validateUrlString = (url: string) => {
   if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) {
     throw new Error('Invalid URL is private ip')
   }
+
+  return true
 }
 
 const contentHandlers: ContentHandler[] = [
diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js
index 8e1a5e1ae..299e689fd 100644
--- a/packages/readabilityjs/Readability.js
+++ b/packages/readabilityjs/Readability.js
@@ -226,7 +226,7 @@ Readability.prototype = {
 
   // These are the classes that readability sets itself.
   CLASSES_TO_PRESERVE: [
-    "page", "twitter-tweet", "tweet-placeholder", "instagram-placeholder", "morning-brew-markets"
+    "page", "twitter-tweet", "tweet-placeholder", "instagram-placeholder", "morning-brew-markets", "prism-code"
   ],
 
   // Classes of placeholder elements that can be empty but shouldn't be removed
diff --git a/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json b/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json
new file mode 100644
index 000000000..0054cf92a
--- /dev/null
+++ b/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json
@@ -0,0 +1,12 @@
+{
+  "title": "How to fetch data with React Hooks",
+  "byline": "Robin Wieruch",
+  "dir": null,
+  "excerpt": "A tutorial on how to fetch data in React with Hooks from third-party APIs. You will use state and effect hooks for the data request from a real API ...",
+  "siteName": null,
+  "siteIcon": "http://fakehost/favicon-32x32.png?v=9db82c76a9aaf54925ac42d41f3d384c",
+  "previewImage": "https://www.robinwieruch.de/static/9b13b3546c675d6f1a1e565f5185cab6/9842e/banner.jpg",
+  "publishedDate": null,
+  "language": "English",
+  "readerable": true
+}
diff --git a/packages/readabilityjs/test/test-pages/robinwieruch.de/expected.html b/packages/readabilityjs/test/test-pages/robinwieruch.de/expected.html
new file mode 100644
index 000000000..0513c6d6c
--- /dev/null
+++ b/packages/readabilityjs/test/test-pages/robinwieruch.de/expected.html
@@ -0,0 +1,1469 @@
+
    
+    
    
+        
    
+            
    
+                
+                
    
+                    
     In this tutorial, I want to show you
+                        how to fetch data in React with Hooks by using the
+                        state and
+                        effect hooks. We will use the widely known
+                        Hacker News API to fetch popular articles from the tech world. You will also implement your custom hook for the data fetching that can be reused anywhere in your application or published on npm as standalone node package.
+                        
+                    
    
+                    
     If you don't know anything about this new React feature, checkout this
+                        . If you want to checkout the finished project for the showcased examples that show how to fetch data in React with Hooks, checkout this
+                        GitHub repository.
+                        
+                    
    
+                    
     If you just want to have a ready to go React Hook for data fetching:
+                        npm install use-data-api and follow the
+                        documentation. Don't forget to star it if you use it :-)
+                        
+                    
    
+                    
    
+                        Note: In the future, React Hooks are not be intended for data fetching in React. Instead, a feature called Suspense will be in charge for it. The following walkthrough is nonetheless a great way to learn more about state and effect hooks in React.
+                        
+                    
    
+                    
    
+                        Data Fetching with React Hooks
+                        
+                    
    
+                    
     If you are not familiar with data fetching in React, checkout my
+                        . It walks you through data fetching with React class components, how it can be made reusable with
+                         and
+                        , and how it deals with error handling and loading spinners. In this article, I want to show you all of it with React Hooks in function components.
+                        
+                    
    
+                    
    
+                    
    import React, { useState } from 'react';
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    return (
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     The App component shows a list of items (hits = Hacker News articles). The state and state update function come from the state hook called
+                        useState that is responsible to manage the local state for the data that we are going to fetch for the App component. The initial state is an empty list of hits in an object that represents the data. No one is setting any state for this data yet.
+                        
+                    
    
+                    
     We are going to use
+                        axios to fetch data, but it is up to you to use another data fetching library or the native fetch API of the browser. If you haven't installed axios yet, you can do so by on the command line with
+                        npm install axios. Then implement your effect hook for the data fetching:
+                        
+                    
    
+                    
    
+                    
    import React, { useState, useEffect } from 'react';
+                    
    
+                    
    import axios from 'axios';
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    useEffect(async () => {
+                    
    
+                    
    const result = await axios(
+                    
    
+                    
    'https://hn.algolia.com/api/v1/search?query=redux',
+                    
    
+                    
    );
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    });
+                    
    
+                    
    return (
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     The effect hook called useEffect is used to fetch the data with axios from the API and to set the data in the local state of the component with the state hook's update function. The promise resolving happens with async/await. 
    
+                    
     However, when you run your application, you should stumble into a nasty loop. The effect hook runs when the component mounts but also when the component updates. Because we are setting the state after every data fetch, the component updates and the effect runs again. It fetches the data again and again. That's a bug and needs to be avoided.
+                        We only want to fetch data when the component mounts. That's why you can provide an empty array as second argument to the effect hook to avoid activating it on component updates but only for the mounting of the component.
+                        
+                    
    
+                    
    
+                    
    import React, { useState, useEffect } from 'react';
+                    
    
+                    
    import axios from 'axios';
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    useEffect(async () => {
+                    
    
+                    
    const result = await axios(
+                    
    
+                    
    'https://hn.algolia.com/api/v1/search?query=redux',
+                    
    
+                    
    );
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    }, []);
+                    
    
+                    
    return (
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     The second argument can be used to define all the variables (allocated in this array) on which the hook depends. If one of the variables changes, the hook runs again. If the array with the variables is empty, the hook doesn't run when updating the component at all, because it doesn't have to watch any variables. 
    
+                    
     There is one last catch. In the code, we are using async/await to fetch data from a third-party API. According to the documentation every function annotated with async returns an implicit promise:
+                        "The async function declaration defines an asynchronous function, which returns an AsyncFunction object. An asynchronous function is a function which operates asynchronously via the event loop, using an implicit Promise to return its result. ". However, an effect hook should return nothing or a clean up function. That's why you may see the following warning in your developer console log:
+                        07:41:22.910 index.js:1452 Warning: useEffect function must return a cleanup function or nothing. Promises and useEffect(async () => ...) are not supported, but you can call an async function inside an effect.. That's why using async directly in the
+                        useEffect function isn't allowed. Let's implement a workaround for it, by using the async function inside the effect.
+                        
+                    
    
+                    
    
+                    
    import React, { useState, useEffect } from 'react';
+                    
    
+                    
    import axios from 'axios';
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    const result = await axios(
+                    
    
+                    
    'https://hn.algolia.com/api/v1/search?query=redux',
+                    
    
+                    
    );
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, []);
+                    
    
+                    
    return (
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     That's data fetching with React hooks in a nutshell. But continue reading if you are interested about error handling, loading indicators, how to trigger the data fetching from a form, and how to implement a reusable data fetching hook. 
    
+                    
    
+                        How to trigger a hook programmatically / manually?
+                        
+                    
    
+                    
     Great, we are fetching data once the component mounts. But what about using an input field to tell the API in which topic we are interested in? "Redux" is taken as default query. But what about topics about "React"? Let's implement an input element to enable someone to fetch other stories than "Redux" stories. Therefore, introduce a new state for the input element. 
    
+                    
    
+                    
    import React, { Fragment, useState, useEffect } from 'react';
+                    
    
+                    
    import axios from 'axios';
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    const result = await axios(
+                    
    
+                    
    'https://hn.algolia.com/api/v1/search?query=redux',
+                    
    
+                    
    );
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, []);
+                    
    
+                    
    return (
+                    
    
+                    
    <Fragment>
+                    
    
+                    
    <input
+                    
    
+                    
    type="text"
+                    
    
+                    
    value={query}
+                    
    
+                    
    onChange={event => setQuery(event.target.value)}
+                    
    
+                    
    />
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    </Fragment>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     At the moment, both states are independent from each other, but now you want to couple them to only fetch articles that are specified by the query in the input field. With the following change, the component should fetch all articles by query term once it mounted. 
    
+                    
    
+                    
    ...
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    const result = await axios(
+                    
    
+                    
    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                    
    
+                    
    );
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, []);
+                    
    
+                    
    return (
+                    
    
+                    
    ...
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     One piece is missing: When you try to type something into the input field, there is no other data fetching after the mounting triggered from the effect. That's because you have provided the empty array as second argument to the effect. The effect depends on no variables, so it is only triggered when the component mounts. However, now the effect should depend on the query. Once the query changes, the data request should fire again. 
    
+                    
    
+                    
    ...
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    const result = await axios(
+                    
    
+                    
    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                    
    
+                    
    );
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, [query]);
+                    
    
+                    
    return (
+                    
    
+                    
    ...
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     The refetching of the data should work once you change the value in the input field. But that opens up another problem: On every character you type into the input field, the effect is triggered and executes another data fetching request. How about providing a button that triggers the request and therefore the hook manually? 
    
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    const [search, setSearch] = useState('');
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    const result = await axios(
+                    
    
+                    
    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                    
    
+                    
    );
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, [query]);
+                    
    
+                    
    return (
+                    
    
+                    
    <Fragment>
+                    
    
+                    
    <input
+                    
    
+                    
    type="text"
+                    
    
+                    
    value={query}
+                    
    
+                    
    onChange={event => setQuery(event.target.value)}
+                    
    
+                    
    />
+                    
    
+                    
    <button type="button" onClick={() => setSearch(query)}>
+                    
    
+                    
    Search
+                    
    
+                    
    </button>
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    </Fragment>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
     Now, make the effect dependant on the search state rather than the fluctuant query state that changes with every key stroke in the input field. Once the user clicks the button, the new search state is set and should trigger the effect hook kinda manually. 
    
+                    
    
+                    
    ...
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    const [search, setSearch] = useState('redux');
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    const result = await axios(
+                    
    
+                    
    `http://hn.algolia.com/api/v1/search?query=${search}`,
+                    
    
+                    
    );
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, [search]);
+                    
    
+                    
    return (
+                    
    
+                    
    ...
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     Also the initial state of the search state is set to the same state as the query state, because the component fetches data also on mount and therefore the result should mirror the value in the input field. However, having a similar query and search state is kinda confusing. Why not set the actual URL as state instead of the search state? 
    
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    const [url, setUrl] = useState(
+                    
    
+                    
    'https://hn.algolia.com/api/v1/search?query=redux',
+                    
    
+                    
    );
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    const result = await axios(url);
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, [url]);
+                    
    
+                    
    return (
+                    
    
+                    
    <Fragment>
+                    
    
+                    
    <input
+                    
    
+                    
    type="text"
+                    
    
+                    
    value={query}
+                    
    
+                    
    onChange={event => setQuery(event.target.value)}
+                    
    
+                    
    />
+                    
    
+                    
    <button
+                    
    
+                    
    type="button"
+                    
    
+                    
    onClick={() =>
+                    
    
+                    
    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                    
    
+                    
    }
+                    
    
+                    
    >
+                    
    
+                    
    Search
+                    
    
+                    
    </button>
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    </Fragment>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
     That's if for the implicit programmatic data fetching with the effect hook. You can decide on which state the effect depends. Once you set this state on a click or in another side-effect, this effect will run again. In this case, if the URL state changes, the effect runs again to fetch stories from the API. 
    
+                    
    
+                        Loading Indicator with React Hooks
+                        
+                    
    
+                    
     Let's introduce a loading indicator to the data fetching. It's just another state that is managed by a state hook. The loading flag is used to render a loading indicator in the App component. 
    
+                    
    
+                    
    import React, { Fragment, useState, useEffect } from 'react';
+                    
    
+                    
    import axios from 'axios';
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    const [url, setUrl] = useState(
+                    
    
+                    
    'https://hn.algolia.com/api/v1/search?query=redux',
+                    
    
+                    
    );
+                    
    
+                    
    const [isLoading, setIsLoading] = useState(false);
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    setIsLoading(true);
+                    
    
+                    
    const result = await axios(url);
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    setIsLoading(false);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, [url]);
+                    
    
+                    
    return (
+                    
    
+                    
    <Fragment>
+                    
    
+                    
    <input
+                    
    
+                    
    type="text"
+                    
    
+                    
    value={query}
+                    
    
+                    
    onChange={event => setQuery(event.target.value)}
+                    
    
+                    
    />
+                    
    
+                    
    <button
+                    
    
+                    
    type="button"
+                    
    
+                    
    onClick={() =>
+                    
    
+                    
    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                    
    
+                    
    }
+                    
    
+                    
    >
+                    
    
+                    
    Search
+                    
    
+                    
    </button>
+                    
    
+                    
    {isLoading ? (
+                    
    
+                    
    <div>Loading ...</div>
+                    
    
+                    
    ) : (
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    )}
+                    
    
+                    
    </Fragment>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     Once the effect is called for data fetching, which happens when the component mounts or the URL state changes, the loading state is set to true. Once the request resolves, the loading state is set to false again. 
    
+                    
    
+                        Error Handling with React Hooks
+                        
+                    
    
+                    
     What about error handling for data fetching with a React hook? The error is just another state initialized with a state hook. Once there is an error state, the App component can render feedback for the user. When using async/await, it is common to use try/catch blocks for error handling. You can do it within the effect: 
    
+                    
    
+                    
    import React, { Fragment, useState, useEffect } from 'react';
+                    
    
+                    
    import axios from 'axios';
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    const [url, setUrl] = useState(
+                    
    
+                    
    'https://hn.algolia.com/api/v1/search?query=redux',
+                    
    
+                    
    );
+                    
    
+                    
    const [isLoading, setIsLoading] = useState(false);
+                    
    
+                    
    const [isError, setIsError] = useState(false);
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    setIsError(false);
+                    
    
+                    
    setIsLoading(true);
+                    
    
+                    
    try {
+                    
    
+                    
    const result = await axios(url);
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    } catch (error) {
+                    
    
+                    
    setIsError(true);
+                    
    
+                    
    }
+                    
    
+                    
    setIsLoading(false);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, [url]);
+                    
    
+                    
    return (
+                    
    
+                    
    <Fragment>
+                    
    
+                    
    <input
+                    
    
+                    
    type="text"
+                    
    
+                    
    value={query}
+                    
    
+                    
    onChange={event => setQuery(event.target.value)}
+                    
    
+                    
    />
+                    
    
+                    
    <button
+                    
    
+                    
    type="button"
+                    
    
+                    
    onClick={() =>
+                    
    
+                    
    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                    
    
+                    
    }
+                    
    
+                    
    >
+                    
    
+                    
    Search
+                    
    
+                    
    </button>
+                    
    
+                    
    {isError && <div>Something went wrong ...</div>}
+                    
    
+                    
    {isLoading ? (
+                    
    
+                    
    <div>Loading ...</div>
+                    
    
+                    
    ) : (
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    )}
+                    
    
+                    
    </Fragment>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     The error state is reset every time the hook runs again. That's useful because after a failed request the user may want to try it again which should reset the error. In order to enforce an error yourself, you can alter the URL into something invalid. Then check whether the error message shows up. 
    
+                    
    
+                        Fetching Data with Forms and React
+                        
+                    
    
+                    
     What about a proper form to fetch data? So far, we have only a combination of input field and button. Once you introduce more input elements, you may want to wrap them with a form element. In addition, a form makes it possible to trigger the button with "Enter" on the keyboard too. 
    
+                    
    
+                    
    function App() {
+                    
    
+                    
    ...
+                    
    
+                    
    return (
+                    
    
+                    
    <Fragment>
+                    
    
+                    
    <form
+                    
    
+                    
    onSubmit={() =>
+                    
    
+                    
    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                    
    
+                    
    }
+                    
    
+                    
    >
+                    
    
+                    
    <input
+                    
    
+                    
    type="text"
+                    
    
+                    
    value={query}
+                    
    
+                    
    onChange={event => setQuery(event.target.value)}
+                    
    
+                    
    />
+                    
    
+                    
    <button type="submit">Search</button>
+                    
    
+                    
    </form>
+                    
    
+                    
    {isError && <div>Something went wrong ...</div>}
+                    
    
+                    
    ...
+                    
    
+                    
    </Fragment>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
     But now the browser reloads when clicking the submit button, because that's the native behavior of the browser when submitting a form. In order to prevent the default behavior, we can invoke a function on the React event. That's how you do it in React class components too. 
    
+                    
    
+                    
    function App() {
+                    
    
+                    
    ...
+                    
    
+                    
    return (
+                    
    
+                    
    <Fragment>
+                    
    
+                    
    <form onSubmit={event => {
+                    
    
+                    
    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`);
+                    
    
+                    
    event.preventDefault();
+                    
    
+                    
    }}>
+                    
    
+                    
    <input
+                    
    
+                    
    type="text"
+                    
    
+                    
    value={query}
+                    
    
+                    
    onChange={event => setQuery(event.target.value)}
+                    
    
+                    
    />
+                    
    
+                    
    <button type="submit">Search</button>
+                    
    
+                    
    </form>
+                    
    
+                    
    {isError && <div>Something went wrong ...</div>}
+                    
    
+                    
    ...
+                    
    
+                    
    </Fragment>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
     Now the browser shouldn't reload anymore when you click the submit button. It works as before, but this time with a form instead of the naive input field and button combination. You can press the "Enter" key on your keyboard too. 
    
+                    
    
+                        Custom Data Fetching Hook
+                        
+                    
    
+                    
     In order to extract a custom hook for data fetching, move everything that belongs to the data fetching, except for the query state that belongs to the input field, but including the loading indicator and error handling, to its own function. Also make sure you return all the necessary variables from the function that are used in the App component. 
    
+                    
    
+                    
    const useHackerNewsApi = () => {
+                    
    
+                    
    const [data, setData] = useState({ hits: [] });
+                    
    
+                    
    const [url, setUrl] = useState(
+                    
    
+                    
    'https://hn.algolia.com/api/v1/search?query=redux',
+                    
    
+                    
    );
+                    
    
+                    
    const [isLoading, setIsLoading] = useState(false);
+                    
    
+                    
    const [isError, setIsError] = useState(false);
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    setIsError(false);
+                    
    
+                    
    setIsLoading(true);
+                    
    
+                    
    try {
+                    
    
+                    
    const result = await axios(url);
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    } catch (error) {
+                    
    
+                    
    setIsError(true);
+                    
    
+                    
    }
+                    
    
+                    
    setIsLoading(false);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, [url]);
+                    
    
+                    
    return [{ data, isLoading, isError }, setUrl];
+                    
    
+                    
    }
+                    
    
+                    
     Now, your new hook can be used in the App component again: 
    
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    const [{ data, isLoading, isError }, doFetch] = useHackerNewsApi();
+                    
    
+                    
    return (
+                    
    
+                    
    <Fragment>
+                    
    
+                    
    <form onSubmit={event => {
+                    
    
+                    
    doFetch(`http://hn.algolia.com/api/v1/search?query=${query}`);
+                    
    
+                    
    event.preventDefault();
+                    
    
+                    
    }}>
+                    
    
+                    
    <input
+                    
    
+                    
    type="text"
+                    
    
+                    
    value={query}
+                    
    
+                    
    onChange={event => setQuery(event.target.value)}
+                    
    
+                    
    />
+                    
    
+                    
    <button type="submit">Search</button>
+                    
    
+                    
    </form>
+                    
    
+                    
    ...
+                    
    
+                    
    </Fragment>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
     The initial state can be made generic too. Pass it simply to the new custom hook: 
    
+                    
    
+                    
    import React, { Fragment, useState, useEffect } from 'react';
+                    
    
+                    
    import axios from 'axios';
+                    
    
+                    
    const useDataApi = (initialUrl, initialData) => {
+                    
    
+                    
    const [data, setData] = useState(initialData);
+                    
    
+                    
    const [url, setUrl] = useState(initialUrl);
+                    
    
+                    
    const [isLoading, setIsLoading] = useState(false);
+                    
    
+                    
    const [isError, setIsError] = useState(false);
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    setIsError(false);
+                    
    
+                    
    setIsLoading(true);
+                    
    
+                    
    try {
+                    
    
+                    
    const result = await axios(url);
+                    
    
+                    
    setData(result.data);
+                    
    
+                    
    } catch (error) {
+                    
    
+                    
    setIsError(true);
+                    
    
+                    
    }
+                    
    
+                    
    setIsLoading(false);
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, [url]);
+                    
    
+                    
    return [{ data, isLoading, isError }, setUrl];
+                    
    
+                    
    };
+                    
    
+                    
    function App() {
+                    
    
+                    
    const [query, setQuery] = useState('redux');
+                    
    
+                    
    const [{ data, isLoading, isError }, doFetch] = useDataApi(
+                    
    
+                    
    'https://hn.algolia.com/api/v1/search?query=redux',
+                    
    
+                    
    { hits: [] },
+                    
    
+                    
    );
+                    
    
+                    
    return (
+                    
    
+                    
    <Fragment>
+                    
    
+                    
    <form
+                    
    
+                    
    onSubmit={event => {
+                    
    
+                    
    doFetch(
+                    
    
+                    
    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                    
    
+                    
    );
+                    
    
+                    
    event.preventDefault();
+                    
    
+                    
    }}
+                    
    
+                    
    >
+                    
    
+                    
    <input
+                    
    
+                    
    type="text"
+                    
    
+                    
    value={query}
+                    
    
+                    
    onChange={event => setQuery(event.target.value)}
+                    
    
+                    
    />
+                    
    
+                    
    <button type="submit">Search</button>
+                    
    
+                    
    </form>
+                    
    
+                    
    {isError && <div>Something went wrong ...</div>}
+                    
    
+                    
    {isLoading ? (
+                    
    
+                    
    <div>Loading ...</div>
+                    
    
+                    
    ) : (
+                    
    
+                    
    <ul>
+                    
    
+                    
    {data.hits.map(item => (
+                    
    
+                    
    <li key={item.objectID}>
+                    
    
+                    
    <a href={item.url}>{item.title}</a>
+                    
    
+                    
    </li>
+                    
    
+                    
    ))}
+                    
    
+                    
    </ul>
+                    
    
+                    
    )}
+                    
    
+                    
    </Fragment>
+                    
    
+                    
    );
+                    
    
+                    
    }
+                    
    
+                    
    export default App;
+                    
    
+                    
     That's it for the data fetching with a custom hook. The hook itself doesn't know anything about the API. It receives all parameters from the outside and only manages necessary states such as the data, loading and error state. It executes the request and returns the data to the component using it as custom data fetching hook. 
    
+                    
    
+                        Reducer Hook for Data Fetching
+                        
+                    
    
+                    
     So far, we have used various state hooks to manage our data fetching state for the data, loading and error state. However, somehow all these states,
+                        . As you can see, they are all used within the data fetching function. A good indicator that they belong together is that they are used one after another (e.g.
+                        setIsError,
+                        setIsLoading). Let's combine all three of them with a
+                         instead.
+                        
+                    
    
+                    
     A Reducer Hook returns us a state object and a function to alter the state object. The function -- called dispatch function -- takes an action which has a type and an optional payload. All this information is used in the actual reducer function to distill a new state from the previous state, the action's optional payload and type. Let's see how this works in code: 
    
+                    
    
+                    
    import React, {
+                    
    
+                    
    Fragment,
+                    
    
+                    
    useState,
+                    
    
+                    
    useEffect,
+                    
    
+                    
    useReducer,
+                    
    
+                    
    } from 'react';
+                    
    
+                    
    import axios from 'axios';
+                    
    
+                    
    const dataFetchReducer = (state, action) => {
+                    
    
+                    
    ...
+                    
    
+                    
    };
+                    
    
+                    
    const useDataApi = (initialUrl, initialData) => {
+                    
    
+                    
    const [url, setUrl] = useState(initialUrl);
+                    
    
+                    
    const [state, dispatch] = useReducer(dataFetchReducer, {
+                    
    
+                    
    isLoading: false,
+                    
    
+                    
    isError: false,
+                    
    
+                    
    data: initialData,
+                    
    
+                    
    });
+                    
    
+                    
    ...
+                    
    
+                    
    };
+                    
    
+                    
     The Reducer Hook takes the reducer function and an initial state object as parameters. In our case, the arguments of the initial states for the data, loading and error state didn't change, but they have been aggregated to one state object managed by one reducer hook instead of single state hooks. 
    
+                    
    
+                    
    const dataFetchReducer = (state, action) => {
+                    
    
+                    
    ...
+                    
    
+                    
    };
+                    
    
+                    
    const useDataApi = (initialUrl, initialData) => {
+                    
    
+                    
    const [url, setUrl] = useState(initialUrl);
+                    
    
+                    
    const [state, dispatch] = useReducer(dataFetchReducer, {
+                    
    
+                    
    isLoading: false,
+                    
    
+                    
    isError: false,
+                    
    
+                    
    data: initialData,
+                    
    
+                    
    });
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    dispatch({ type: 'FETCH_INIT' });
+                    
    
+                    
    try {
+                    
    
+                    
    const result = await axios(url);
+                    
    
+                    
    dispatch({ type: 'FETCH_SUCCESS', payload: result.data });
+                    
    
+                    
    } catch (error) {
+                    
    
+                    
    dispatch({ type: 'FETCH_FAILURE' });
+                    
    
+                    
    }
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    }, [url]);
+                    
    
+                    
    ...
+                    
    
+                    
    };
+                    
    
+                    
     Now, when fetching data, the dispatch function can be used to send information to the reducer function. The object being send with the dispatch function has a mandatory
+                        type property and an optional
+                        payload property. The type tells the reducer function which state transition needs to be applied and the payload can additionally be used by the reducer to distill the new state. After all, we only have three state transitions: initializing the fetching process, notifying about a successful data fetching result, and notifying about an erroneous data fetching result.
+                        
+                    
    
+                    
     In the end of the custom hook, the state is returned as before, but because we have a state object and not the standalone states anymore. This way, the one who calls the
+                        useDataApi custom hook still gets access to
+                        data,
+                        isLoading and
+                        isError:
+                        
+                    
    
+                    
    
+                    
    const useDataApi = (initialUrl, initialData) => {
+                    
    
+                    
    const [url, setUrl] = useState(initialUrl);
+                    
    
+                    
    const [state, dispatch] = useReducer(dataFetchReducer, {
+                    
    
+                    
    isLoading: false,
+                    
    
+                    
    isError: false,
+                    
    
+                    
    data: initialData,
+                    
    
+                    
    });
+                    
    
+                    
    ...
+                    
    
+                    
    return [state, setUrl];
+                    
    
+                    
    };
+                    
    
+                    
     Last but not least, the implementation of the reducer function is missing. It needs to act on three different state transitions called
+                        FETCH_INIT,
+                        FETCH_SUCCESS and
+                        FETCH_FAILURE. Each state transition needs to return a new state object. Let's see how this can be implemented with a switch case statement:
+                        
+                    
    
+                    
    
+                    
    const dataFetchReducer = (state, action) => {
+                    
    
+                    
    switch (action.type) {
+                    
    
+                    
    case 'FETCH_INIT':
+                    
    
+                    
    return { ...state };
+                    
    
+                    
    case 'FETCH_SUCCESS':
+                    
    
+                    
    return { ...state };
+                    
    
+                    
    case 'FETCH_FAILURE':
+                    
    
+                    
    return { ...state };
+                    
    
+                    
    default:
+                    
    
+                    
    throw new Error();
+                    
    
+                    
    }
+                    
    
+                    
    };
+                    
    
+                    
     A reducer function has access to the current state and the incoming action via its arguments. So far, in out switch case statement each state transition only returns the previous state. A destructuring statement is used to keep the state object immutable -- meaning the state is never directly mutated -- to enforce best practices. Now let's override a few of the current's state returned properties to alter the state with each state transition: 
    
+                    
    
+                    
    const dataFetchReducer = (state, action) => {
+                    
    
+                    
    switch (action.type) {
+                    
    
+                    
    case 'FETCH_INIT':
+                    
    
+                    
    return {
+                    
    
+                    
    ...state,
+                    
    
+                    
    isLoading: true,
+                    
    
+                    
    isError: false
+                    
    
+                    
    };
+                    
    
+                    
    case 'FETCH_SUCCESS':
+                    
    
+                    
    return {
+                    
    
+                    
    ...state,
+                    
    
+                    
    isLoading: false,
+                    
    
+                    
    isError: false,
+                    
    
+                    
    data: action.payload,
+                    
    
+                    
    };
+                    
    
+                    
    case 'FETCH_FAILURE':
+                    
    
+                    
    return {
+                    
    
+                    
    ...state,
+                    
    
+                    
    isLoading: false,
+                    
    
+                    
    isError: true,
+                    
    
+                    
    };
+                    
    
+                    
    default:
+                    
    
+                    
    throw new Error();
+                    
    
+                    
    }
+                    
    
+                    
    };
+                    
    
+                    
     Now every state transition, decided by the action's type, returns a new state based on the previous state and the optional payload. For instance, in the case of a successful request, the payload is used to set the data of the new state object. 
    
+                    
     In conclusion, the Reducer Hook makes sure that this portion of the state management is encapsulated with its own logic. By providing action types and optional payloads, you will always end up with a predicatbale state change. In addition, you will never run into invalid states. For instance, previously it would have been possible to accidently set the
+                        isLoading and
+                        isError states to true. What should be displayed in the UI for this case? Now, each state transition defined by the reducer function leads to a valid state object.
+                        
+                    
    
+                    
    
+                        Abort Data Fetching in Effect Hook
+                        
+                    
    
+                    
     It's a common problem in React that component state is set even though the component got already unmounted (e.g. due to navigating away with React Router). I have written about this issue previously over here which describes
+                         in various scenarios. Let's see how we can prevent to set state in our custom hook for the data fetching:
+                        
+                    
    
+                    
    
+                    
    const useDataApi = (initialUrl, initialData) => {
+                    
    
+                    
    const [url, setUrl] = useState(initialUrl);
+                    
    
+                    
    const [state, dispatch] = useReducer(dataFetchReducer, {
+                    
    
+                    
    isLoading: false,
+                    
    
+                    
    isError: false,
+                    
    
+                    
    data: initialData,
+                    
    
+                    
    });
+                    
    
+                    
    useEffect(() => {
+                    
    
+                    
    let didCancel = false;
+                    
    
+                    
    const fetchData = async () => {
+                    
    
+                    
    dispatch({ type: 'FETCH_INIT' });
+                    
    
+                    
    try {
+                    
    
+                    
    const result = await axios(url);
+                    
    
+                    
    if (!didCancel) {
+                    
    
+                    
    dispatch({ type: 'FETCH_SUCCESS', payload: result.data });
+                    
    
+                    
    }
+                    
    
+                    
    } catch (error) {
+                    
    
+                    
    if (!didCancel) {
+                    
    
+                    
    dispatch({ type: 'FETCH_FAILURE' });
+                    
    
+                    
    }
+                    
    
+                    
    }
+                    
    
+                    
    };
+                    
    
+                    
    fetchData();
+                    
    
+                    
    return () => {
+                    
    
+                    
    didCancel = true;
+                    
    
+                    
    };
+                    
    
+                    
    }, [url]);
+                    
    
+                    
    return [state, setUrl];
+                    
    
+                    
    };
+                    
    
+                    
     Every Effect Hook comes with a clean up function which runs when a component unmounts. The clean up function is the one function returned from the hook. In our case, we use a boolean flag called
+                        didCancel to let our data fetching logic know about the state (mounted/unmounted) of the component. If the component did unmount, the flag should be set to
+                        true which results in preventing to set the component state after the data fetching has been asynchronously resolved eventually.
+                        
+                    
    
+                    
    
+                        Note: Actually not the data fetching is aborted -- which could be achieved with
+                            Axios Cancellation -- but the state transition is not performed anymore for the unmounted component. Since Axios Cancellation has not the best API in my eyes, this boolean flag to prevent setting state does the job as well.
+                            
+                        
+                    
    
+                    
    
+                    
     You have learned how the React hooks for state and effects can be used in React for data fetching. If you are curious about data fetching in class components (and function components) with render props and higher-order components, checkout out my other article from the beginning. Otherwise, I hope this article was useful to you for learning about React Hooks and how to use them in a real world scenario. 
    
+                
    
+            
    
+        
    
+        
    
+            
    
+                Keep reading about
+                     
+                    
+                
+            
    
+            
    
+                
    
+                    
    
+                        
    
+                            
    
+                                
+                            
    
+                        
    
+                        
     Every once in a while we need to test API requests. Axios is one of the most popular JavaScript libraries to fetch data from remote APIs . Hence, we will use Axios for our data fetching example… 
    
+                    
    
+                
    
+                
    
+                    
    
+                        
    
+                            
    
+                                
+                            
    
+                        
    
+                        
     React introduced Hooks quite a while ago. With their release, Hooks gave function components the ability to use state and side-effects with built-in Hooks such as React's useState Hook and… 
    
+                    
    
+                
    
+            
    
+        
    
+        
    
+            
     The Road to React 
    
+            
     Learn React by building real world applications. No setup configuration. No tooling. Plain React in 200+ pages of learning material. Learn React like 50.000+ readers. 
    
+            
    Get it on Amazon.
+            
    
+        
    
+        
    
+            
    
+                
+            
    
+            
    
+                
+            
    
+            
    
+                
+            
    
+            
    
+                
+            
    
+            
    
+                
+            
    
+            
    
+            
    
+            
    
+        
    
+    
    
+
    
\ No newline at end of file
diff --git a/packages/readabilityjs/test/test-pages/robinwieruch.de/source.html b/packages/readabilityjs/test/test-pages/robinwieruch.de/source.html
new file mode 100644
index 000000000..e62263bbc
--- /dev/null
+++ b/packages/readabilityjs/test/test-pages/robinwieruch.de/source.html
@@ -0,0 +1,2504 @@
+
+
+    
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+            How to fetch data with React Hooks
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+    
+    
+        
    
+            
    
+                
    
+                    
+                    
+                    
    
+                        
    
+                            
    
+                                
    
+                                    How to fetch data with React Hooks
+                                
    
+                                
+                                
    
+                                    
+                                    
+                                    
+                                    
+                                     by Robin Wieruch
+                                    
+                                    
    
+                                        
+                                    
     - Edit this Post
+                                
    
+                                
+                            
    
+                            
+                            
    
+                                
    
+                                    In this tutorial, I want to show you how to fetch data in React with Hooks by using the state and effect hooks. We will use the widely known Hacker News API to fetch popular articles from the tech world. You will also implement your custom hook for the data fetching that can be reused anywhere in your application or published on npm as standalone node package.
+                                
    
+                                
    
+                                    If you don't know anything about this new React feature, checkout this . If you want to checkout the finished project for the showcased examples that show how to fetch data in React with Hooks, checkout this GitHub repository.
+                                
    
+                                
    
+                                    If you just want to have a ready to go React Hook for data fetching: npm install use-data-api and follow the documentation. Don't forget to star it if you use it :-)
+                                
    
+                                
    
+                                    Note: In the future, React Hooks are not be intended for data fetching in React. Instead, a feature called Suspense will be in charge for it. The following walkthrough is nonetheless a great way to learn more about state and effect hooks in React.
+                                
    
+                                
    
+                                    Data Fetching with React Hooks
+                                
    
+                                
    
+                                    If you are not familiar with data fetching in React, checkout my . It walks you through data fetching with React class components, how it can be made reusable with  and , and how it deals with error handling and loading spinners. In this article, I want to show you all of it with React Hooks in function components.
+                                
    
+                                
    
+                                
    
+                                    import React, { useState } from 'react';
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    The App component shows a list of items (hits = Hacker News articles). The state and state update function come from the state hook called useState that is responsible to manage the local state for the data that we are going to fetch for the App component. The initial state is an empty list of hits in an object that represents the data. No one is setting any state for this data yet.
+                                
    
+                                
    
+                                    We are going to use axios to fetch data, but it is up to you to use another data fetching library or the native fetch API of the browser. If you haven't installed axios yet, you can do so by on the command line with npm install axios. Then implement your effect hook for the data fetching:
+                                
    
+                                
    
+                                
    
+                                    import React, { useState, useEffect } from 'react';
+                                
    
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                
    
+                                    useEffect(async () => {
+                                
    
+                                
    
+                                    const result = await axios(
+                                
    
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    });
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    The effect hook called useEffect is used to fetch the data with axios from the API and to set the data in the local state of the component with the state hook's update function. The promise resolving happens with async/await.
+                                
    
+                                
    
+                                    However, when you run your application, you should stumble into a nasty loop. The effect hook runs when the component mounts but also when the component updates. Because we are setting the state after every data fetch, the component updates and the effect runs again. It fetches the data again and again. That's a bug and needs to be avoided. We only want to fetch data when the component mounts. That's why you can provide an empty array as second argument to the effect hook to avoid activating it on component updates but only for the mounting of the component.
+                                
    
+                                
    
+                                
    
+                                    import React, { useState, useEffect } from 'react';
+                                
    
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                
    
+                                    useEffect(async () => {
+                                
    
+                                
    
+                                    const result = await axios(
+                                
    
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    }, []);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    The second argument can be used to define all the variables (allocated in this array) on which the hook depends. If one of the variables changes, the hook runs again. If the array with the variables is empty, the hook doesn't run when updating the component at all, because it doesn't have to watch any variables.
+                                
    
+                                
    
+                                    There is one last catch. In the code, we are using async/await to fetch data from a third-party API. According to the documentation every function annotated with async returns an implicit promise: "The async function declaration defines an asynchronous function, which returns an AsyncFunction object. An asynchronous function is a function which operates asynchronously via the event loop, using an implicit Promise to return its result. ". However, an effect hook should return nothing or a clean up function. That's why you may see the following warning in your developer console log: 07:41:22.910 index.js:1452 Warning: useEffect function must return a cleanup function or nothing. Promises and useEffect(async () => ...) are not supported, but you can call an async function inside an effect.. That's why using async directly in the useEffect function isn't allowed. Let's implement a workaround for it, by using the async function inside the effect.
+                                
    
+                                
    
+                                
    
+                                    import React, { useState, useEffect } from 'react';
+                                
    
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    const result = await axios(
+                                
    
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, []);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    That's data fetching with React hooks in a nutshell. But continue reading if you are interested about error handling, loading indicators, how to trigger the data fetching from a form, and how to implement a reusable data fetching hook.
+                                
    
+                                
    
+                                    How to trigger a hook programmatically / manually?
+                                
    
+                                
    
+                                    Great, we are fetching data once the component mounts. But what about using an input field to tell the API in which topic we are interested in? "Redux" is taken as default query. But what about topics about "React"? Let's implement an input element to enable someone to fetch other stories than "Redux" stories. Therefore, introduce a new state for the input element.
+                                
    
+                                
    
+                                
    
+                                    import React, { Fragment, useState, useEffect } from 'react';
+                                
    
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    const result = await axios(
+                                
    
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, []);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <Fragment>
+                                
    
+                                
    
+                                    <input
+                                
    
+                                
    
+                                    type="text"
+                                
    
+                                
    
+                                    value={query}
+                                
    
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                
    
+                                    />
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    </Fragment>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    At the moment, both states are independent from each other, but now you want to couple them to only fetch articles that are specified by the query in the input field. With the following change, the component should fetch all articles by query term once it mounted.
+                                
    
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    const result = await axios(
+                                
    
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, []);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    One piece is missing: When you try to type something into the input field, there is no other data fetching after the mounting triggered from the effect. That's because you have provided the empty array as second argument to the effect. The effect depends on no variables, so it is only triggered when the component mounts. However, now the effect should depend on the query. Once the query changes, the data request should fire again.
+                                
    
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    const result = await axios(
+                                
    
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, [query]);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    The refetching of the data should work once you change the value in the input field. But that opens up another problem: On every character you type into the input field, the effect is triggered and executes another data fetching request. How about providing a button that triggers the request and therefore the hook manually?
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                    const [search, setSearch] = useState('');
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    const result = await axios(
+                                
    
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, [query]);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <Fragment>
+                                
    
+                                
    
+                                    <input
+                                
    
+                                
    
+                                    type="text"
+                                
    
+                                
    
+                                    value={query}
+                                
    
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                
    
+                                    />
+                                
    
+                                
    
+                                    <button type="button" onClick={() => setSearch(query)}>
+                                
    
+                                
    
+                                    Search
+                                
    
+                                
    
+                                    </button>
+                                
    
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    </Fragment>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    Now, make the effect dependant on the search state rather than the fluctuant query state that changes with every key stroke in the input field. Once the user clicks the button, the new search state is set and should trigger the effect hook kinda manually.
+                                
    
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                    const [search, setSearch] = useState('redux');
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    const result = await axios(
+                                
    
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${search}`,
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, [search]);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    Also the initial state of the search state is set to the same state as the query state, because the component fetches data also on mount and therefore the result should mirror the value in the input field. However, having a similar query and search state is kinda confusing. Why not set the actual URL as state instead of the search state?
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                    const [url, setUrl] = useState(
+                                
    
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, [url]);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <Fragment>
+                                
    
+                                
    
+                                    <input
+                                
    
+                                
    
+                                    type="text"
+                                
    
+                                
    
+                                    value={query}
+                                
    
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                
    
+                                    />
+                                
    
+                                
    
+                                    <button
+                                
    
+                                
    
+                                    type="button"
+                                
    
+                                
    
+                                    onClick={() =>
+                                
    
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    >
+                                
    
+                                
    
+                                    Search
+                                
    
+                                
    
+                                    </button>
+                                
    
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    </Fragment>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    That's if for the implicit programmatic data fetching with the effect hook. You can decide on which state the effect depends. Once you set this state on a click or in another side-effect, this effect will run again. In this case, if the URL state changes, the effect runs again to fetch stories from the API.
+                                
    
+                                
    
+                                    Loading Indicator with React Hooks
+                                
    
+                                
    
+                                    Let's introduce a loading indicator to the data fetching. It's just another state that is managed by a state hook. The loading flag is used to render a loading indicator in the App component.
+                                
    
+                                
    
+                                
    
+                                    import React, { Fragment, useState, useEffect } from 'react';
+                                
    
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                    const [url, setUrl] = useState(
+                                
    
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    const [isLoading, setIsLoading] = useState(false);
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    setIsLoading(true);
+                                
    
+                                
    
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    setIsLoading(false);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, [url]);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <Fragment>
+                                
    
+                                
    
+                                    <input
+                                
    
+                                
    
+                                    type="text"
+                                
    
+                                
    
+                                    value={query}
+                                
    
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                
    
+                                    />
+                                
    
+                                
    
+                                    <button
+                                
    
+                                
    
+                                    type="button"
+                                
    
+                                
    
+                                    onClick={() =>
+                                
    
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    >
+                                
    
+                                
    
+                                    Search
+                                
    
+                                
    
+                                    </button>
+                                
    
+                                
    
+                                
    
+                                    {isLoading ? (
+                                
    
+                                
    
+                                    <div>Loading ...</div>
+                                
    
+                                
    
+                                    ) : (
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    )}
+                                
    
+                                
    
+                                    </Fragment>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    Once the effect is called for data fetching, which happens when the component mounts or the URL state changes, the loading state is set to true. Once the request resolves, the loading state is set to false again.
+                                
    
+                                
    
+                                    Error Handling with React Hooks
+                                
    
+                                
    
+                                    What about error handling for data fetching with a React hook? The error is just another state initialized with a state hook. Once there is an error state, the App component can render feedback for the user. When using async/await, it is common to use try/catch blocks for error handling. You can do it within the effect:
+                                
    
+                                
    
+                                
    
+                                    import React, { Fragment, useState, useEffect } from 'react';
+                                
    
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                    const [url, setUrl] = useState(
+                                
    
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    const [isLoading, setIsLoading] = useState(false);
+                                
    
+                                
    
+                                    const [isError, setIsError] = useState(false);
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    setIsError(false);
+                                
    
+                                
    
+                                    setIsLoading(true);
+                                
    
+                                
    
+                                
    
+                                    try {
+                                
    
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    } catch (error) {
+                                
    
+                                
    
+                                    setIsError(true);
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    setIsLoading(false);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, [url]);
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <Fragment>
+                                
    
+                                
    
+                                    <input
+                                
    
+                                
    
+                                    type="text"
+                                
    
+                                
    
+                                    value={query}
+                                
    
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                
    
+                                    />
+                                
    
+                                
    
+                                    <button
+                                
    
+                                
    
+                                    type="button"
+                                
    
+                                
    
+                                    onClick={() =>
+                                
    
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    >
+                                
    
+                                
    
+                                    Search
+                                
    
+                                
    
+                                    </button>
+                                
    
+                                
    
+                                
    
+                                    {isError && <div>Something went wrong ...</div>}
+                                
    
+                                
    
+                                
    
+                                    {isLoading ? (
+                                
    
+                                
    
+                                    <div>Loading ...</div>
+                                
    
+                                
    
+                                    ) : (
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    )}
+                                
    
+                                
    
+                                    </Fragment>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    The error state is reset every time the hook runs again. That's useful because after a failed request the user may want to try it again which should reset the error. In order to enforce an error yourself, you can alter the URL into something invalid. Then check whether the error message shows up.
+                                
    
+                                
    
+                                    Fetching Data with Forms and React
+                                
    
+                                
    
+                                    What about a proper form to fetch data? So far, we have only a combination of input field and button. Once you introduce more input elements, you may want to wrap them with a form element. In addition, a form makes it possible to trigger the button with "Enter" on the keyboard too.
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <Fragment>
+                                
    
+                                
    
+                                    <form
+                                
    
+                                
    
+                                    onSubmit={() =>
+                                
    
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    >
+                                
    
+                                
    
+                                    <input
+                                
    
+                                
    
+                                    type="text"
+                                
    
+                                
    
+                                    value={query}
+                                
    
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                
    
+                                    />
+                                
    
+                                
    
+                                    <button type="submit">Search</button>
+                                
    
+                                
    
+                                    </form>
+                                
    
+                                
    
+                                
    
+                                    {isError && <div>Something went wrong ...</div>}
+                                
    
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    </Fragment>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    But now the browser reloads when clicking the submit button, because that's the native behavior of the browser when submitting a form. In order to prevent the default behavior, we can invoke a function on the React event. That's how you do it in React class components too.
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <Fragment>
+                                
    
+                                
    
+                                    <form onSubmit={event => {
+                                
    
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`);
+                                
    
+                                
    
+                                
    
+                                    event.preventDefault();
+                                
    
+                                
    
+                                    }}>
+                                
    
+                                
    
+                                    <input
+                                
    
+                                
    
+                                    type="text"
+                                
    
+                                
    
+                                    value={query}
+                                
    
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                
    
+                                    />
+                                
    
+                                
    
+                                    <button type="submit">Search</button>
+                                
    
+                                
    
+                                    </form>
+                                
    
+                                
    
+                                
    
+                                    {isError && <div>Something went wrong ...</div>}
+                                
    
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    </Fragment>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    Now the browser shouldn't reload anymore when you click the submit button. It works as before, but this time with a form instead of the naive input field and button combination. You can press the "Enter" key on your keyboard too.
+                                
    
+                                
    
+                                    Custom Data Fetching Hook
+                                
    
+                                
    
+                                    In order to extract a custom hook for data fetching, move everything that belongs to the data fetching, except for the query state that belongs to the input field, but including the loading indicator and error handling, to its own function. Also make sure you return all the necessary variables from the function that are used in the App component.
+                                
    
+                                
    
+                                
    
+                                    const useHackerNewsApi = () => {
+                                
    
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                
    
+                                    const [url, setUrl] = useState(
+                                
    
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    const [isLoading, setIsLoading] = useState(false);
+                                
    
+                                
    
+                                    const [isError, setIsError] = useState(false);
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    setIsError(false);
+                                
    
+                                
    
+                                    setIsLoading(true);
+                                
    
+                                
    
+                                
    
+                                    try {
+                                
    
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    } catch (error) {
+                                
    
+                                
    
+                                    setIsError(true);
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    setIsLoading(false);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, [url]);
+                                
    
+                                
    
+                                
    
+                                    return [{ data, isLoading, isError }, setUrl];
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    Now, your new hook can be used in the App component again:
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                    const [{ data, isLoading, isError }, doFetch] = useHackerNewsApi();
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <Fragment>
+                                
    
+                                
    
+                                    <form onSubmit={event => {
+                                
    
+                                
    
+                                    doFetch(`http://hn.algolia.com/api/v1/search?query=${query}`);
+                                
    
+                                
    
+                                
    
+                                    event.preventDefault();
+                                
    
+                                
    
+                                    }}>
+                                
    
+                                
    
+                                    <input
+                                
    
+                                
    
+                                    type="text"
+                                
    
+                                
    
+                                    value={query}
+                                
    
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                
    
+                                    />
+                                
    
+                                
    
+                                    <button type="submit">Search</button>
+                                
    
+                                
    
+                                    </form>
+                                
    
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    </Fragment>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    The initial state can be made generic too. Pass it simply to the new custom hook:
+                                
    
+                                
    
+                                
    
+                                    import React, { Fragment, useState, useEffect } from 'react';
+                                
    
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                
    
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                
    
+                                    const [data, setData] = useState(initialData);
+                                
    
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                
    
+                                    const [isLoading, setIsLoading] = useState(false);
+                                
    
+                                
    
+                                    const [isError, setIsError] = useState(false);
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    setIsError(false);
+                                
    
+                                
    
+                                    setIsLoading(true);
+                                
    
+                                
    
+                                
    
+                                    try {
+                                
    
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                
    
+                                
    
+                                    setData(result.data);
+                                
    
+                                
    
+                                    } catch (error) {
+                                
    
+                                
    
+                                    setIsError(true);
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    setIsLoading(false);
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, [url]);
+                                
    
+                                
    
+                                
    
+                                    return [{ data, isLoading, isError }, setUrl];
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    function App() {
+                                
    
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                
    
+                                    const [{ data, isLoading, isError }, doFetch] = useDataApi(
+                                
    
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                
    
+                                    { hits: [] },
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    return (
+                                
    
+                                
    
+                                    <Fragment>
+                                
    
+                                
    
+                                    <form
+                                
    
+                                
    
+                                    onSubmit={event => {
+                                
    
+                                
    
+                                    doFetch(
+                                
    
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                
    
+                                    event.preventDefault();
+                                
    
+                                
    
+                                    }}
+                                
    
+                                
    
+                                    >
+                                
    
+                                
    
+                                    <input
+                                
    
+                                
    
+                                    type="text"
+                                
    
+                                
    
+                                    value={query}
+                                
    
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                
    
+                                    />
+                                
    
+                                
    
+                                    <button type="submit">Search</button>
+                                
    
+                                
    
+                                    </form>
+                                
    
+                                
    
+                                
    
+                                    {isError && <div>Something went wrong ...</div>}
+                                
    
+                                
    
+                                
    
+                                    {isLoading ? (
+                                
    
+                                
    
+                                    <div>Loading ...</div>
+                                
    
+                                
    
+                                    ) : (
+                                
    
+                                
    
+                                    <ul>
+                                
    
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                
    
+                                    </li>
+                                
    
+                                
    
+                                    ))}
+                                
    
+                                
    
+                                    </ul>
+                                
    
+                                
    
+                                    )}
+                                
    
+                                
    
+                                    </Fragment>
+                                
    
+                                
    
+                                    );
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                
    
+                                    export default App;
+                                
    
+                                
    
+                                    That's it for the data fetching with a custom hook. The hook itself doesn't know anything about the API. It receives all parameters from the outside and only manages necessary states such as the data, loading and error state. It executes the request and returns the data to the component using it as custom data fetching hook.
+                                
    
+                                
    
+                                    Reducer Hook for Data Fetching
+                                
    
+                                
    
+                                    So far, we have used various state hooks to manage our data fetching state for the data, loading and error state. However, somehow all these states, . As you can see, they are all used within the data fetching function. A good indicator that they belong together is that they are used one after another (e.g. setIsError, setIsLoading). Let's combine all three of them with a  instead.
+                                
    
+                                
    
+                                    A Reducer Hook returns us a state object and a function to alter the state object. The function -- called dispatch function -- takes an action which has a type and an optional payload. All this information is used in the actual reducer function to distill a new state from the previous state, the action's optional payload and type. Let's see how this works in code:
+                                
    
+                                
    
+                                
    
+                                    import React, {
+                                
    
+                                
    
+                                    Fragment,
+                                
    
+                                
    
+                                    useState,
+                                
    
+                                
    
+                                    useEffect,
+                                
    
+                                
    
+                                    useReducer,
+                                
    
+                                
    
+                                    } from 'react';
+                                
    
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                
    
+                                
    
+                                    const dataFetchReducer = (state, action) => {
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                
    
+                                
    
+                                    const [state, dispatch] = useReducer(dataFetchReducer, {
+                                
    
+                                
    
+                                    isLoading: false,
+                                
    
+                                
    
+                                    isError: false,
+                                
    
+                                
    
+                                    data: initialData,
+                                
    
+                                
    
+                                    });
+                                
    
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    The Reducer Hook takes the reducer function and an initial state object as parameters. In our case, the arguments of the initial states for the data, loading and error state didn't change, but they have been aggregated to one state object managed by one reducer hook instead of single state hooks.
+                                
    
+                                
    
+                                
    
+                                    const dataFetchReducer = (state, action) => {
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                
    
+                                
    
+                                    const [state, dispatch] = useReducer(dataFetchReducer, {
+                                
    
+                                
    
+                                    isLoading: false,
+                                
    
+                                
    
+                                    isError: false,
+                                
    
+                                
    
+                                    data: initialData,
+                                
    
+                                
    
+                                    });
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    dispatch({ type: 'FETCH_INIT' });
+                                
    
+                                
    
+                                
    
+                                    try {
+                                
    
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                
    
+                                
    
+                                    dispatch({ type: 'FETCH_SUCCESS', payload: result.data });
+                                
    
+                                
    
+                                    } catch (error) {
+                                
    
+                                
    
+                                    dispatch({ type: 'FETCH_FAILURE' });
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                    }, [url]);
+                                
    
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    Now, when fetching data, the dispatch function can be used to send information to the reducer function. The object being send with the dispatch function has a mandatory type property and an optional payload property. The type tells the reducer function which state transition needs to be applied and the payload can additionally be used by the reducer to distill the new state. After all, we only have three state transitions: initializing the fetching process, notifying about a successful data fetching result, and notifying about an erroneous data fetching result.
+                                
    
+                                
    
+                                    In the end of the custom hook, the state is returned as before, but because we have a state object and not the standalone states anymore. This way, the one who calls the useDataApi custom hook still gets access to data, isLoading and isError:
+                                
    
+                                
    
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                
    
+                                
    
+                                    const [state, dispatch] = useReducer(dataFetchReducer, {
+                                
    
+                                
    
+                                    isLoading: false,
+                                
    
+                                
    
+                                    isError: false,
+                                
    
+                                
    
+                                    data: initialData,
+                                
    
+                                
    
+                                    });
+                                
    
+                                
    
+                                
    
+                                    ...
+                                
    
+                                
    
+                                
    
+                                    return [state, setUrl];
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    Last but not least, the implementation of the reducer function is missing. It needs to act on three different state transitions called FETCH_INIT, FETCH_SUCCESS and FETCH_FAILURE. Each state transition needs to return a new state object. Let's see how this can be implemented with a switch case statement:
+                                
    
+                                
    
+                                
    
+                                    const dataFetchReducer = (state, action) => {
+                                
    
+                                
    
+                                    switch (action.type) {
+                                
    
+                                
    
+                                    case 'FETCH_INIT':
+                                
    
+                                
    
+                                    return { ...state };
+                                
    
+                                
    
+                                    case 'FETCH_SUCCESS':
+                                
    
+                                
    
+                                    return { ...state };
+                                
    
+                                
    
+                                    case 'FETCH_FAILURE':
+                                
    
+                                
    
+                                    return { ...state };
+                                
    
+                                
    
+                                    default:
+                                
    
+                                
    
+                                    throw new Error();
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    A reducer function has access to the current state and the incoming action via its arguments. So far, in out switch case statement each state transition only returns the previous state. A destructuring statement is used to keep the state object immutable -- meaning the state is never directly mutated -- to enforce best practices. Now let's override a few of the current's state returned properties to alter the state with each state transition:
+                                
    
+                                
    
+                                
    
+                                    const dataFetchReducer = (state, action) => {
+                                
    
+                                
    
+                                    switch (action.type) {
+                                
    
+                                
    
+                                    case 'FETCH_INIT':
+                                
    
+                                
    
+                                    return {
+                                
    
+                                
    
+                                    ...state,
+                                
    
+                                
    
+                                    isLoading: true,
+                                
    
+                                
    
+                                    isError: false
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    case 'FETCH_SUCCESS':
+                                
    
+                                
    
+                                    return {
+                                
    
+                                
    
+                                    ...state,
+                                
    
+                                
    
+                                    isLoading: false,
+                                
    
+                                
    
+                                    isError: false,
+                                
    
+                                
    
+                                    data: action.payload,
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    case 'FETCH_FAILURE':
+                                
    
+                                
    
+                                    return {
+                                
    
+                                
    
+                                    ...state,
+                                
    
+                                
    
+                                    isLoading: false,
+                                
    
+                                
    
+                                    isError: true,
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    default:
+                                
    
+                                
    
+                                    throw new Error();
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    Now every state transition, decided by the action's type, returns a new state based on the previous state and the optional payload. For instance, in the case of a successful request, the payload is used to set the data of the new state object.
+                                
    
+                                
    
+                                    In conclusion, the Reducer Hook makes sure that this portion of the state management is encapsulated with its own logic. By providing action types and optional payloads, you will always end up with a predicatbale state change. In addition, you will never run into invalid states. For instance, previously it would have been possible to accidently set the isLoading and isError states to true. What should be displayed in the UI for this case? Now, each state transition defined by the reducer function leads to a valid state object.
+                                
    
+                                
    
+                                    Abort Data Fetching in Effect Hook
+                                
    
+                                
    
+                                    It's a common problem in React that component state is set even though the component got already unmounted (e.g. due to navigating away with React Router). I have written about this issue previously over here which describes  in various scenarios. Let's see how we can prevent to set state in our custom hook for the data fetching:
+                                
    
+                                
    
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                
    
+                                
    
+                                    const [state, dispatch] = useReducer(dataFetchReducer, {
+                                
    
+                                
    
+                                    isLoading: false,
+                                
    
+                                
    
+                                    isError: false,
+                                
    
+                                
    
+                                    data: initialData,
+                                
    
+                                
    
+                                    });
+                                
    
+                                
    
+                                
    
+                                    useEffect(() => {
+                                
    
+                                
    
+                                    let didCancel = false;
+                                
    
+                                
    
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                
    
+                                    dispatch({ type: 'FETCH_INIT' });
+                                
    
+                                
    
+                                
    
+                                    try {
+                                
    
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                
    
+                                
    
+                                    if (!didCancel) {
+                                
    
+                                
    
+                                    dispatch({ type: 'FETCH_SUCCESS', payload: result.data });
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    } catch (error) {
+                                
    
+                                
    
+                                    if (!didCancel) {
+                                
    
+                                
    
+                                    dispatch({ type: 'FETCH_FAILURE' });
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    }
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                
    
+                                    fetchData();
+                                
    
+                                
    
+                                
    
+                                    return () => {
+                                
    
+                                
    
+                                    didCancel = true;
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    }, [url]);
+                                
    
+                                
    
+                                
    
+                                    return [state, setUrl];
+                                
    
+                                
    
+                                    };
+                                
    
+                                
    
+                                    Every Effect Hook comes with a clean up function which runs when a component unmounts. The clean up function is the one function returned from the hook. In our case, we use a boolean flag called didCancel to let our data fetching logic know about the state (mounted/unmounted) of the component. If the component did unmount, the flag should be set to true which results in preventing to set the component state after the data fetching has been asynchronously resolved eventually.
+                                
    
+                                
    
+                                    Note: Actually not the data fetching is aborted -- which could be achieved with Axios Cancellation -- but the state transition is not performed anymore for the unmounted component. Since Axios Cancellation has not the best API in my eyes, this boolean flag to prevent setting state does the job as well.
+                                
    
+                                
    
+                                
    
+                                    You have learned how the React hooks for state and effects can be used in React for data fetching. If you are curious about data fetching in class components (and function components) with render props and higher-order components, checkout out my other article from the beginning. Otherwise, I hope this article was useful to you for learning about React Hooks and how to use them in a real world scenario.
+                                
    
+                            
    
+                            
    
+                                
    
+                                    Discuss on TwitterShare on Twitter
+                                
    
+                            
    
+                        
    
+                    
    
+                    
    
+                        
    
+                            
    
+                                Keep reading about 
+                            
    
+                            
    
+                                
    
+                                    
    
+                                        
+                                    
    
+                                    
    
+                                        
    
+                                            
    
+                                                
+                                            
    
+                                        
    
+                                        
    
+                                            Every once in a while we need to test API requests. Axios is one of the most popular JavaScript libraries to fetch data from remote APIs . Hence, we will use Axios for our data fetching example…
+                                        
    
+                                    
    
+                                
    
+                                
    
+                                    
    
+                                        
+                                    
    
+                                    
    
+                                        
    
+                                            
    
+                                                
+                                            
    
+                                        
    
+                                        
    
+                                            React introduced Hooks quite a while ago. With their release, Hooks gave function components the ability to use state and side-effects with built-in Hooks such as React's useState Hook and…
+                                        
    
+                                    
    
+                                
    
+                            
    
+                        
    
+                    
    
+                    
    
+                        
    
+                            
    
+                                
+                            
    
+                            
    
+                                
    
+                                    The Road to React
+                                
    
+                                
    
+                                    Learn React by building real world applications. No setup configuration. No tooling. Plain React in 200+ pages of learning material. Learn React like 50.000+ readers.
+                                
    
+                                
    
+                                
+                                
    Get it on Amazon.
+                            
    
+                        
    
+                    
    
+                    
    
+                        
    
+                            
    
+                                
+                            
    
+                            
    
+                                
+                            
    
+                        
    
+                        
    
+                            
+                        
    
+                        
    
+                            
+                        
    
+                        
    
+                            
+                        
    
+                        
    
+                            
+                        
    
+                        
    
+                            
+                        
    
+                        
    
+                        
    
+                    
    
+                    
+                
    
+            
    
+            
    
+        
    
+        
+        
+        
+        
+        
+        
+        
+    
+
diff --git a/packages/readabilityjs/test/test-pages/robinwieruch.de/url.txt b/packages/readabilityjs/test/test-pages/robinwieruch.de/url.txt
new file mode 100644
index 000000000..fb86eba6b
--- /dev/null
+++ b/packages/readabilityjs/test/test-pages/robinwieruch.de/url.txt
@@ -0,0 +1 @@
+https://www.robinwieruch.de/react-hooks-fetch-data/
\ No newline at end of file

From 7fc468b2bd42f549983ca3a97a7535eb6a957c79 Mon Sep 17 00:00:00 2001
From: sywhb 
Date: Tue, 14 Feb 2023 04:34:50 +0000
Subject: [PATCH 7/7] Update generated html

---
 packages/readabilityjs/test/index.html        |    6 +
 .../test-pages/robinwieruch.de/distiller.html | 1315 +++++++++++++++++
 2 files changed, 1321 insertions(+)
 create mode 100644 packages/readabilityjs/test/test-pages/robinwieruch.de/distiller.html

diff --git a/packages/readabilityjs/test/index.html b/packages/readabilityjs/test/index.html
index a6b272a3f..4813f55dc 100644
--- a/packages/readabilityjs/test/index.html
+++ b/packages/readabilityjs/test/index.html
@@ -236,6 +236,12 @@
         [dom-distiller]
       
    • 
   
+      
  • robinwieruch.de
    
+        [source]
+        [readability]
+        [dom-distiller]
+      
    • 
+  
       
  • github-blog
    
         [source]
         [readability]
diff --git a/packages/readabilityjs/test/test-pages/robinwieruch.de/distiller.html b/packages/readabilityjs/test/test-pages/robinwieruch.de/distiller.html
new file mode 100644
index 000000000..3fcb0294a
--- /dev/null
+++ b/packages/readabilityjs/test/test-pages/robinwieruch.de/distiller.html
@@ -0,0 +1,1315 @@
+
    
+                                    In this tutorial, I want to show you how to fetch data in React with Hooks by using the state and effect hooks. We will use the widely known Hacker News API to fetch popular articles from the tech world. You will also implement your custom hook for the data fetching that can be reused anywhere in your application or published on npm as standalone node package.
+                                
    
+                                    If you don't know anything about this new React feature, checkout this introduction to React Hooks. If you want to checkout the finished project for the showcased examples that show how to fetch data in React with Hooks, checkout this GitHub repository.
+                                
    
+                                    If you just want to have a ready to go React Hook for data fetching: npm install use-data-api and follow the documentation. Don't forget to star it if you use it :-)
+                                
    
+                                    Note: In the future, React Hooks are not be intended for data fetching in React. Instead, a feature called Suspense will be in charge for it. The following walkthrough is nonetheless a great way to learn more about state and effect hooks in React.
+                                
    
+                                    Data Fetching with React Hooks
+                                
    
+                                    If you are not familiar with data fetching in React, checkout my extensive data fetching in React article. It walks you through data fetching with React class components, how it can be made reusable with Render Prop Components and Higher-Order Components, and how it deals with error handling and loading spinners. In this article, I want to show you all of it with React Hooks in function components.
+                                
    
+                                    import React, { useState } from 'react';
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    return (
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    The App component shows a list of items (hits = Hacker News articles). The state and state update function come from the state hook called useState that is responsible to manage the local state for the data that we are going to fetch for the App component. The initial state is an empty list of hits in an object that represents the data. No one is setting any state for this data yet.
+                                
    
+                                    We are going to use axios to fetch data, but it is up to you to use another data fetching library or the native fetch API of the browser. If you haven't installed axios yet, you can do so by on the command line with npm install axios. Then implement your effect hook for the data fetching:
+                                
    
+                                    import React, { useState, useEffect } from 'react';
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    useEffect(async () => {
+                                
    
+                                    const result = await axios(
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                    );
+                                
    
+                                    setData(result.data);
+                                
    
+                                    });
+                                
    
+                                    return (
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    The effect hook called useEffect is used to fetch the data with axios from the API and to set the data in the local state of the component with the state hook's update function. The promise resolving happens with async/await.
+                                
    
+                                    However, when you run your application, you should stumble into a nasty loop. The effect hook runs when the component mounts but also when the component updates. Because we are setting the state after every data fetch, the component updates and the effect runs again. It fetches the data again and again. That's a bug and needs to be avoided. We only want to fetch data when the component mounts. That's why you can provide an empty array as second argument to the effect hook to avoid activating it on component updates but only for the mounting of the component.
+                                
    
+                                    import React, { useState, useEffect } from 'react';
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    useEffect(async () => {
+                                
    
+                                    const result = await axios(
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                    );
+                                
    
+                                    setData(result.data);
+                                
    
+                                    }, []);
+                                
    
+                                    return (
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    The second argument can be used to define all the variables (allocated in this array) on which the hook depends. If one of the variables changes, the hook runs again. If the array with the variables is empty, the hook doesn't run when updating the component at all, because it doesn't have to watch any variables.
+                                
    
+                                    There is one last catch. In the code, we are using async/await to fetch data from a third-party API. According to the documentation every function annotated with async returns an implicit promise: "The async function declaration defines an asynchronous function, which returns an AsyncFunction object. An asynchronous function is a function which operates asynchronously via the event loop, using an implicit Promise to return its result. ". However, an effect hook should return nothing or a clean up function. That's why you may see the following warning in your developer console log: 07:41:22.910 index.js:1452 Warning: useEffect function must return a cleanup function or nothing. Promises and useEffect(async () => ...) are not supported, but you can call an async function inside an effect.. That's why using async directly in the useEffect function isn't allowed. Let's implement a workaround for it, by using the async function inside the effect.
+                                
    
+                                    import React, { useState, useEffect } from 'react';
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    const result = await axios(
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                    );
+                                
    
+                                    setData(result.data);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, []);
+                                
    
+                                    return (
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    That's data fetching with React hooks in a nutshell. But continue reading if you are interested about error handling, loading indicators, how to trigger the data fetching from a form, and how to implement a reusable data fetching hook.
+                                
    
+                                    How to trigger a hook programmatically / manually?
+                                
    
+                                    Great, we are fetching data once the component mounts. But what about using an input field to tell the API in which topic we are interested in? "Redux" is taken as default query. But what about topics about "React"? Let's implement an input element to enable someone to fetch other stories than "Redux" stories. Therefore, introduce a new state for the input element.
+                                
    
+                                    import React, { Fragment, useState, useEffect } from 'react';
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    const result = await axios(
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                    );
+                                
    
+                                    setData(result.data);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, []);
+                                
    
+                                    return (
+                                
    
+                                    <Fragment>
+                                
    
+                                    <input
+                                
    
+                                    type="text"
+                                
    
+                                    value={query}
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                    />
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    </Fragment>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    At the moment, both states are independent from each other, but now you want to couple them to only fetch articles that are specified by the query in the input field. With the following change, the component should fetch all articles by query term once it mounted.
+                                
    
+                                    ...
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    const result = await axios(
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                                
    
+                                    );
+                                
    
+                                    setData(result.data);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, []);
+                                
    
+                                    return (
+                                
    
+                                    ...
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    One piece is missing: When you try to type something into the input field, there is no other data fetching after the mounting triggered from the effect. That's because you have provided the empty array as second argument to the effect. The effect depends on no variables, so it is only triggered when the component mounts. However, now the effect should depend on the query. Once the query changes, the data request should fire again.
+                                
    
+                                    ...
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    const result = await axios(
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                                
    
+                                    );
+                                
    
+                                    setData(result.data);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, [query]);
+                                
    
+                                    return (
+                                
    
+                                    ...
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    The refetching of the data should work once you change the value in the input field. But that opens up another problem: On every character you type into the input field, the effect is triggered and executes another data fetching request. How about providing a button that triggers the request and therefore the hook manually?
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    const [search, setSearch] = useState('');
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    const result = await axios(
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                                
    
+                                    );
+                                
    
+                                    setData(result.data);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, [query]);
+                                
    
+                                    return (
+                                
    
+                                    <Fragment>
+                                
    
+                                    <input
+                                
    
+                                    type="text"
+                                
    
+                                    value={query}
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                    />
+                                
    
+                                    <button type="button" onClick={() => setSearch(query)}>
+                                
    
+                                    Search
+                                
    
+                                    </button>
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    </Fragment>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    Now, make the effect dependant on the search state rather than the fluctuant query state that changes with every key stroke in the input field. Once the user clicks the button, the new search state is set and should trigger the effect hook kinda manually.
+                                
    
+                                    ...
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    const [search, setSearch] = useState('redux');
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    const result = await axios(
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${search}`,
+                                
    
+                                    );
+                                
    
+                                    setData(result.data);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, [search]);
+                                
    
+                                    return (
+                                
    
+                                    ...
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    Also the initial state of the search state is set to the same state as the query state, because the component fetches data also on mount and therefore the result should mirror the value in the input field. However, having a similar query and search state is kinda confusing. Why not set the actual URL as state instead of the search state?
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    const [url, setUrl] = useState(
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                    );
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                    setData(result.data);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, [url]);
+                                
    
+                                    return (
+                                
    
+                                    <Fragment>
+                                
    
+                                    <input
+                                
    
+                                    type="text"
+                                
    
+                                    value={query}
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                    />
+                                
    
+                                    <button
+                                
    
+                                    type="button"
+                                
    
+                                    onClick={() =>
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                                
    
+                                    }
+                                
    
+                                    >
+                                
    
+                                    Search
+                                
    
+                                    </button>
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    </Fragment>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    That's if for the implicit programmatic data fetching with the effect hook. You can decide on which state the effect depends. Once you set this state on a click or in another side-effect, this effect will run again. In this case, if the URL state changes, the effect runs again to fetch stories from the API.
+                                
    
+                                    Loading Indicator with React Hooks
+                                
    
+                                    Let's introduce a loading indicator to the data fetching. It's just another state that is managed by a state hook. The loading flag is used to render a loading indicator in the App component.
+                                
    
+                                    import React, { Fragment, useState, useEffect } from 'react';
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    const [url, setUrl] = useState(
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                    );
+                                
    
+                                    const [isLoading, setIsLoading] = useState(false);
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    setIsLoading(true);
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                    setData(result.data);
+                                
    
+                                    setIsLoading(false);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, [url]);
+                                
    
+                                    return (
+                                
    
+                                    <Fragment>
+                                
    
+                                    <input
+                                
    
+                                    type="text"
+                                
    
+                                    value={query}
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                    />
+                                
    
+                                    <button
+                                
    
+                                    type="button"
+                                
    
+                                    onClick={() =>
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                                
    
+                                    }
+                                
    
+                                    >
+                                
    
+                                    Search
+                                
    
+                                    </button>
+                                
    
+                                    {isLoading ? (
+                                
    
+                                    <div>Loading ...</div>
+                                
    
+                                    ) : (
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    )}
+                                
    
+                                    </Fragment>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    Once the effect is called for data fetching, which happens when the component mounts or the URL state changes, the loading state is set to true. Once the request resolves, the loading state is set to false again.
+                                
    
+                                    Error Handling with React Hooks
+                                
    
+                                    What about error handling for data fetching with a React hook? The error is just another state initialized with a state hook. Once there is an error state, the App component can render feedback for the user. When using async/await, it is common to use try/catch blocks for error handling. You can do it within the effect:
+                                
    
+                                    import React, { Fragment, useState, useEffect } from 'react';
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                    function App() {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    const [url, setUrl] = useState(
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                    );
+                                
    
+                                    const [isLoading, setIsLoading] = useState(false);
+                                
    
+                                    const [isError, setIsError] = useState(false);
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    setIsError(false);
+                                
    
+                                    setIsLoading(true);
+                                
    
+                                    try {
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                    setData(result.data);
+                                
    
+                                    } catch (error) {
+                                
    
+                                    setIsError(true);
+                                
    
+                                    }
+                                
    
+                                    setIsLoading(false);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, [url]);
+                                
    
+                                    return (
+                                
    
+                                    <Fragment>
+                                
    
+                                    <input
+                                
    
+                                    type="text"
+                                
    
+                                    value={query}
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                    />
+                                
    
+                                    <button
+                                
    
+                                    type="button"
+                                
    
+                                    onClick={() =>
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                                
    
+                                    }
+                                
    
+                                    >
+                                
    
+                                    Search
+                                
    
+                                    </button>
+                                
    
+                                    {isError && <div>Something went wrong ...</div>}
+                                
    
+                                    {isLoading ? (
+                                
    
+                                    <div>Loading ...</div>
+                                
    
+                                    ) : (
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    )}
+                                
    
+                                    </Fragment>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    The error state is reset every time the hook runs again. That's useful because after a failed request the user may want to try it again which should reset the error. In order to enforce an error yourself, you can alter the URL into something invalid. Then check whether the error message shows up.
+                                
    
+                                    Fetching Data with Forms and React
+                                
    
+                                    What about a proper form to fetch data? So far, we have only a combination of input field and button. Once you introduce more input elements, you may want to wrap them with a form element. In addition, a form makes it possible to trigger the button with "Enter" on the keyboard too.
+                                
    
+                                    function App() {
+                                
    
+                                    ...
+                                
    
+                                    return (
+                                
    
+                                    <Fragment>
+                                
    
+                                    <form
+                                
    
+                                    onSubmit={() =>
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`)
+                                
    
+                                    }
+                                
    
+                                    >
+                                
    
+                                    <input
+                                
    
+                                    type="text"
+                                
    
+                                    value={query}
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                    />
+                                
    
+                                    <button type="submit">Search</button>
+                                
    
+                                    </form>
+                                
    
+                                    {isError && <div>Something went wrong ...</div>}
+                                
    
+                                    ...
+                                
    
+                                    </Fragment>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    But now the browser reloads when clicking the submit button, because that's the native behavior of the browser when submitting a form. In order to prevent the default behavior, we can invoke a function on the React event. That's how you do it in React class components too.
+                                
    
+                                    function App() {
+                                
    
+                                    ...
+                                
    
+                                    return (
+                                
    
+                                    <Fragment>
+                                
    
+                                    <form onSubmit={event => {
+                                
    
+                                    setUrl(`http://hn.algolia.com/api/v1/search?query=${query}`);
+                                
    
+                                    event.preventDefault();
+                                
    
+                                    }}>
+                                
    
+                                    <input
+                                
    
+                                    type="text"
+                                
    
+                                    value={query}
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                    />
+                                
    
+                                    <button type="submit">Search</button>
+                                
    
+                                    </form>
+                                
    
+                                    {isError && <div>Something went wrong ...</div>}
+                                
    
+                                    ...
+                                
    
+                                    </Fragment>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    Now the browser shouldn't reload anymore when you click the submit button. It works as before, but this time with a form instead of the naive input field and button combination. You can press the "Enter" key on your keyboard too.
+                                
    
+                                    Custom Data Fetching Hook
+                                
    
+                                    In order to extract a custom hook for data fetching, move everything that belongs to the data fetching, except for the query state that belongs to the input field, but including the loading indicator and error handling, to its own function. Also make sure you return all the necessary variables from the function that are used in the App component.
+                                
    
+                                    const useHackerNewsApi = () => {
+                                
    
+                                    const [data, setData] = useState({ hits: [] });
+                                
    
+                                    const [url, setUrl] = useState(
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                    );
+                                
    
+                                    const [isLoading, setIsLoading] = useState(false);
+                                
    
+                                    const [isError, setIsError] = useState(false);
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    setIsError(false);
+                                
    
+                                    setIsLoading(true);
+                                
    
+                                    try {
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                    setData(result.data);
+                                
    
+                                    } catch (error) {
+                                
    
+                                    setIsError(true);
+                                
    
+                                    }
+                                
    
+                                    setIsLoading(false);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, [url]);
+                                
    
+                                    return [{ data, isLoading, isError }, setUrl];
+                                
    
+                                    }
+                                
    
+                                    Now, your new hook can be used in the App component again:
+                                
    
+                                    function App() {
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    const [{ data, isLoading, isError }, doFetch] = useHackerNewsApi();
+                                
    
+                                    return (
+                                
    
+                                    <Fragment>
+                                
    
+                                    <form onSubmit={event => {
+                                
    
+                                    doFetch(`http://hn.algolia.com/api/v1/search?query=${query}`);
+                                
    
+                                    event.preventDefault();
+                                
    
+                                    }}>
+                                
    
+                                    <input
+                                
    
+                                    type="text"
+                                
    
+                                    value={query}
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                    />
+                                
    
+                                    <button type="submit">Search</button>
+                                
    
+                                    </form>
+                                
    
+                                    ...
+                                
    
+                                    </Fragment>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    The initial state can be made generic too. Pass it simply to the new custom hook:
+                                
    
+                                    import React, { Fragment, useState, useEffect } from 'react';
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                    const [data, setData] = useState(initialData);
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                    const [isLoading, setIsLoading] = useState(false);
+                                
    
+                                    const [isError, setIsError] = useState(false);
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    setIsError(false);
+                                
    
+                                    setIsLoading(true);
+                                
    
+                                    try {
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                    setData(result.data);
+                                
    
+                                    } catch (error) {
+                                
    
+                                    setIsError(true);
+                                
    
+                                    }
+                                
    
+                                    setIsLoading(false);
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, [url]);
+                                
    
+                                    return [{ data, isLoading, isError }, setUrl];
+                                
    
+                                    };
+                                
    
+                                    function App() {
+                                
    
+                                    const [query, setQuery] = useState('redux');
+                                
    
+                                    const [{ data, isLoading, isError }, doFetch] = useDataApi(
+                                
    
+                                    'https://hn.algolia.com/api/v1/search?query=redux',
+                                
    
+                                    { hits: [] },
+                                
    
+                                    );
+                                
    
+                                    return (
+                                
    
+                                    <Fragment>
+                                
    
+                                    <form
+                                
    
+                                    onSubmit={event => {
+                                
    
+                                    doFetch(
+                                
    
+                                    `http://hn.algolia.com/api/v1/search?query=${query}`,
+                                
    
+                                    );
+                                
    
+                                    event.preventDefault();
+                                
    
+                                    }}
+                                
    
+                                    >
+                                
    
+                                    <input
+                                
    
+                                    type="text"
+                                
    
+                                    value={query}
+                                
    
+                                    onChange={event => setQuery(event.target.value)}
+                                
    
+                                    />
+                                
    
+                                    <button type="submit">Search</button>
+                                
    
+                                    </form>
+                                
    
+                                    {isError && <div>Something went wrong ...</div>}
+                                
    
+                                    {isLoading ? (
+                                
    
+                                    <div>Loading ...</div>
+                                
    
+                                    ) : (
+                                
    
+                                    <ul>
+                                
    
+                                    {data.hits.map(item => (
+                                
    
+                                    <li key={item.objectID}>
+                                
    
+                                    <a href={item.url}>{item.title}</a>
+                                
    
+                                    </li>
+                                
    
+                                    ))}
+                                
    
+                                    </ul>
+                                
    
+                                    )}
+                                
    
+                                    </Fragment>
+                                
    
+                                    );
+                                
    
+                                    }
+                                
    
+                                    export default App;
+                                
    
+                                    That's it for the data fetching with a custom hook. The hook itself doesn't know anything about the API. It receives all parameters from the outside and only manages necessary states such as the data, loading and error state. It executes the request and returns the data to the component using it as custom data fetching hook.
+                                
    
+                                    Reducer Hook for Data Fetching
+                                
    
+                                    So far, we have used various state hooks to manage our data fetching state for the data, loading and error state. However, somehow all these states, managed with their own state hook, belong together because they care about the same cause. As you can see, they are all used within the data fetching function. A good indicator that they belong together is that they are used one after another (e.g. setIsError, setIsLoading). Let's combine all three of them with a Reducer Hook instead.
+                                
    
+                                    A Reducer Hook returns us a state object and a function to alter the state object. The function -- called dispatch function -- takes an action which has a type and an optional payload. All this information is used in the actual reducer function to distill a new state from the previous state, the action's optional payload and type. Let's see how this works in code:
+                                
    
+                                    import React, {
+                                
    
+                                    Fragment,
+                                
    
+                                    useState,
+                                
    
+                                    useEffect,
+                                
    
+                                    useReducer,
+                                
    
+                                    } from 'react';
+                                
    
+                                    import axios from 'axios';
+                                
    
+                                    const dataFetchReducer = (state, action) => {
+                                
    
+                                    ...
+                                
    
+                                    };
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                    const [state, dispatch] = useReducer(dataFetchReducer, {
+                                
    
+                                    isLoading: false,
+                                
    
+                                    isError: false,
+                                
    
+                                    data: initialData,
+                                
    
+                                    });
+                                
    
+                                    ...
+                                
    
+                                    };
+                                
    
+                                    The Reducer Hook takes the reducer function and an initial state object as parameters. In our case, the arguments of the initial states for the data, loading and error state didn't change, but they have been aggregated to one state object managed by one reducer hook instead of single state hooks.
+                                
    
+                                    const dataFetchReducer = (state, action) => {
+                                
    
+                                    ...
+                                
    
+                                    };
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                    const [state, dispatch] = useReducer(dataFetchReducer, {
+                                
    
+                                    isLoading: false,
+                                
    
+                                    isError: false,
+                                
    
+                                    data: initialData,
+                                
    
+                                    });
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    dispatch({ type: 'FETCH_INIT' });
+                                
    
+                                    try {
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                    dispatch({ type: 'FETCH_SUCCESS', payload: result.data });
+                                
    
+                                    } catch (error) {
+                                
    
+                                    dispatch({ type: 'FETCH_FAILURE' });
+                                
    
+                                    }
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    }, [url]);
+                                
    
+                                    ...
+                                
    
+                                    };
+                                
    
+                                    Now, when fetching data, the dispatch function can be used to send information to the reducer function. The object being send with the dispatch function has a mandatory type property and an optional payload property. The type tells the reducer function which state transition needs to be applied and the payload can additionally be used by the reducer to distill the new state. After all, we only have three state transitions: initializing the fetching process, notifying about a successful data fetching result, and notifying about an erroneous data fetching result.
+                                
    
+                                    In the end of the custom hook, the state is returned as before, but because we have a state object and not the standalone states anymore. This way, the one who calls the useDataApi custom hook still gets access to data, isLoading and isError:
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                    const [state, dispatch] = useReducer(dataFetchReducer, {
+                                
    
+                                    isLoading: false,
+                                
    
+                                    isError: false,
+                                
    
+                                    data: initialData,
+                                
    
+                                    });
+                                
    
+                                    ...
+                                
    
+                                    return [state, setUrl];
+                                
    
+                                    };
+                                
    
+                                    Last but not least, the implementation of the reducer function is missing. It needs to act on three different state transitions called FETCH_INIT, FETCH_SUCCESS and FETCH_FAILURE. Each state transition needs to return a new state object. Let's see how this can be implemented with a switch case statement:
+                                
    
+                                    const dataFetchReducer = (state, action) => {
+                                
    
+                                    switch (action.type) {
+                                
    
+                                    case 'FETCH_INIT':
+                                
    
+                                    return { ...state };
+                                
    
+                                    case 'FETCH_SUCCESS':
+                                
    
+                                    return { ...state };
+                                
    
+                                    case 'FETCH_FAILURE':
+                                
    
+                                    return { ...state };
+                                
    
+                                    default:
+                                
    
+                                    throw new Error();
+                                
    
+                                    }
+                                
    
+                                    };
+                                
    
+                                    A reducer function has access to the current state and the incoming action via its arguments. So far, in out switch case statement each state transition only returns the previous state. A destructuring statement is used to keep the state object immutable -- meaning the state is never directly mutated -- to enforce best practices. Now let's override a few of the current's state returned properties to alter the state with each state transition:
+                                
    
+                                    const dataFetchReducer = (state, action) => {
+                                
    
+                                    switch (action.type) {
+                                
    
+                                    case 'FETCH_INIT':
+                                
    
+                                    return {
+                                
    
+                                    ...state,
+                                
    
+                                    isLoading: true,
+                                
    
+                                    isError: false
+                                
    
+                                    };
+                                
    
+                                    case 'FETCH_SUCCESS':
+                                
    
+                                    return {
+                                
    
+                                    ...state,
+                                
    
+                                    isLoading: false,
+                                
    
+                                    isError: false,
+                                
    
+                                    data: action.payload,
+                                
    
+                                    };
+                                
    
+                                    case 'FETCH_FAILURE':
+                                
    
+                                    return {
+                                
    
+                                    ...state,
+                                
    
+                                    isLoading: false,
+                                
    
+                                    isError: true,
+                                
    
+                                    };
+                                
    
+                                    default:
+                                
    
+                                    throw new Error();
+                                
    
+                                    }
+                                
    
+                                    };
+                                
    
+                                    Now every state transition, decided by the action's type, returns a new state based on the previous state and the optional payload. For instance, in the case of a successful request, the payload is used to set the data of the new state object.
+                                
    
+                                    In conclusion, the Reducer Hook makes sure that this portion of the state management is encapsulated with its own logic. By providing action types and optional payloads, you will always end up with a predicatbale state change. In addition, you will never run into invalid states. For instance, previously it would have been possible to accidently set the isLoading and isError states to true. What should be displayed in the UI for this case? Now, each state transition defined by the reducer function leads to a valid state object.
+                                
    
+                                    Abort Data Fetching in Effect Hook
+                                
    
+                                    It's a common problem in React that component state is set even though the component got already unmounted (e.g. due to navigating away with React Router). I have written about this issue previously over here which describes how to prevent setting state for unmounted components in various scenarios. Let's see how we can prevent to set state in our custom hook for the data fetching:
+                                
    
+                                    const useDataApi = (initialUrl, initialData) => {
+                                
    
+                                    const [url, setUrl] = useState(initialUrl);
+                                
    
+                                    const [state, dispatch] = useReducer(dataFetchReducer, {
+                                
    
+                                    isLoading: false,
+                                
    
+                                    isError: false,
+                                
    
+                                    data: initialData,
+                                
    
+                                    });
+                                
    
+                                    useEffect(() => {
+                                
    
+                                    let didCancel = false;
+                                
    
+                                    const fetchData = async () => {
+                                
    
+                                    dispatch({ type: 'FETCH_INIT' });
+                                
    
+                                    try {
+                                
    
+                                    const result = await axios(url);
+                                
    
+                                    if (!didCancel) {
+                                
    
+                                    dispatch({ type: 'FETCH_SUCCESS', payload: result.data });
+                                
    
+                                    }
+                                
    
+                                    } catch (error) {
+                                
    
+                                    if (!didCancel) {
+                                
    
+                                    dispatch({ type: 'FETCH_FAILURE' });
+                                
    
+                                    }
+                                
    
+                                    }
+                                
    
+                                    };
+                                
    
+                                    fetchData();
+                                
    
+                                    return () => {
+                                
    
+                                    didCancel = true;
+                                
    
+                                    };
+                                
    
+                                    }, [url]);
+                                
    
+                                    return [state, setUrl];
+                                
    
+                                    };
+                                
    
+                                    Every Effect Hook comes with a clean up function which runs when a component unmounts. The clean up function is the one function returned from the hook. In our case, we use a boolean flag called didCancel to let our data fetching logic know about the state (mounted/unmounted) of the component. If the component did unmount, the flag should be set to true which results in preventing to set the component state after the data fetching has been asynchronously resolved eventually.
+                                
    
+                                    Note: Actually not the data fetching is aborted -- which could be achieved with Axios Cancellation -- but the state transition is not performed anymore for the unmounted component. Since Axios Cancellation has not the best API in my eyes, this boolean flag to prevent setting state does the job as well.
+                                
    
+                                    You have learned how the React hooks for state and effects can be used in React for data fetching. If you are curious about data fetching in class components (and function components) with render props and higher-order components, checkout out my other article from the beginning. Otherwise, I hope this article was useful to you for learning about React Hooks and how to use them in a real world scenario.
+                                
    
\ No newline at end of file