mirror of
https://github.com/arfct/itty-bitty.git
synced 2026-03-11 08:54:33 +00:00
Add microdata parser
This commit is contained in:
Nicholas Jitkoff
parent
b8d5440e9e
commit
59b690be6c
5 changed files with 276 additions and 52 deletions
|
|
@ -172,6 +172,17 @@ function parseBittyURL(url) {
|
|||
return {path, hashTitle, hashData}
|
||||
}
|
||||
|
||||
async function testDecode(rawData) {
|
||||
console.group("Testing decode")
|
||||
let t1 = performance.now()
|
||||
let gz = dataToBase64TD(rawData)
|
||||
let t2 = performance.now()
|
||||
console.log("textdecode", gz.length, Math.round((t2 - t1)*1000));
|
||||
let xz = await dataToBase64FR(rawData);
|
||||
let t3 = performance.now()
|
||||
console.log("reader", xz.length, Math.round((t3 - t2)*1000));
|
||||
}
|
||||
|
||||
async function testCompression(rawData) {
|
||||
console.group("Testing compression")
|
||||
let t1 = performance.now()
|
||||
|
|
@ -442,7 +453,11 @@ async function hashString(string, base = 36) {
|
|||
|
||||
|
||||
function dataToBase64(data) {
|
||||
return btoa(String.fromCharCode(...new Uint8Array(data)));
|
||||
return btoa(String.fromCharCode.apply(null, new Uint8Array(data)));
|
||||
}
|
||||
|
||||
function dataToBase64TD(data) {
|
||||
return btoa(new TextDecoder('utf8').decode(data));
|
||||
}
|
||||
|
||||
function dataToBase64FR(data) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
body {
|
||||
font-family: sans-serif;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
|
||||
margin:0;
|
||||
}
|
||||
|
||||
|
|
@ -214,3 +214,38 @@ body:not(.loading) #loader {
|
|||
90% {box-shadow:var(--ld18),var(--ld2), var(--ld6),var(--ld10),var(--ld14)}
|
||||
95% {box-shadow:var(--ld19),var(--ld3), var(--ld7),var(--ld11),var(--ld15)}
|
||||
}
|
||||
|
||||
|
||||
dialog {
|
||||
border: 2px solid white;
|
||||
border-radius: 4px;
|
||||
box-shadow: 0 0 0 2px black, inset 0 0 0 3px black;
|
||||
min-width: 320px;
|
||||
padding: 1.2em;
|
||||
}
|
||||
dialog, dialog button {font-size: 18px;font-weight: 600;font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";}
|
||||
|
||||
dialog::backdrop {
|
||||
background: repeating-conic-gradient(rgba(128,128,128,.5) 0% 25%, transparent 0% 50%) 50% / 4px 4px;
|
||||
}
|
||||
|
||||
|
||||
dialog button {
|
||||
border: 1px solid white;
|
||||
border-radius: 9px;
|
||||
box-shadow: inset 0 0 0 2px black;
|
||||
min-width: 4em;
|
||||
min-height: 1.8em;
|
||||
background-color: white;
|
||||
margin-left:1em;
|
||||
padding: 0 0.8em;
|
||||
}
|
||||
dialog button:last-child {
|
||||
box-shadow: 0 0 0 4px black, inset 0 0 0 2px black;
|
||||
}
|
||||
|
||||
dialog form {
|
||||
text-align: right;
|
||||
margin-top: 1em;
|
||||
font-size: 18px;
|
||||
}
|
||||
|
|
@ -61,12 +61,7 @@
|
|||
document.getElementById("favicon").href = 'data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em"><text y=".9em">'+ favicon + '</text></svg>'
|
||||
}
|
||||
|
||||
window.el = function (tagName, attrs, ...children) {
|
||||
let l = document.createElement(tagName);
|
||||
Object.entries(attrs).forEach(([k,v]) => l[k] = v);
|
||||
children.forEach((c) => l.appendChild(c));
|
||||
return l;
|
||||
}
|
||||
window.el = bitty.el;
|
||||
|
||||
const renderers = {
|
||||
"application/ld+json": {script:"/render/recipe.html"},
|
||||
|
|
@ -85,8 +80,7 @@
|
|||
|
||||
function share(info) { // {title, text, url}
|
||||
if (!info.url) info = {title:document.title, text:document.title, url:location.href};
|
||||
console.log("Share", info);
|
||||
|
||||
|
||||
if (navigator.share) {
|
||||
navigator.share(info)
|
||||
.then(() => { console.log('Shared!');})
|
||||
|
|
@ -196,11 +190,26 @@
|
|||
renderContent();
|
||||
}
|
||||
}
|
||||
|
||||
if (e.data.error) {
|
||||
showError(e.data.error)
|
||||
}
|
||||
if (e.data.setStorage) document.localStorage.setItem(contentHash, e.data.set);
|
||||
if (e.data.getStorage) document.getElementById("iframe").postMessage(document.localStorage.getItem(contentHash), e.origin)
|
||||
}, false);
|
||||
|
||||
|
||||
function showError(error) {
|
||||
console.warn("🛑", error)
|
||||
let dialog = el("dialog",
|
||||
el("div", {}, error),
|
||||
el("form", {method: "dialog"},
|
||||
|
||||
el("button", {onclick:history.back.bind(history)}, "Learn More"),
|
||||
el("button", {onclick:history.back.bind(history)}, "Go Back")));
|
||||
document.body.appendChild(dialog)
|
||||
dialog.showModal();
|
||||
}
|
||||
|
||||
async function renderContent() {
|
||||
|
||||
showLoader(true)
|
||||
|
|
|
|||
|
|
@ -1,42 +1,199 @@
|
|||
// let title = params.title
|
||||
|
||||
|
||||
parent.postMessage({title:"Parsing Content..."}, "*");
|
||||
document.body.appendChild(document.createTextNode("• • •"))
|
||||
|
||||
let lca = (a,b) => {
|
||||
return a.parents().has(b).first();
|
||||
}
|
||||
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(params.body, "text/html");
|
||||
console.log("🛠️ Parsing Document ", doc)
|
||||
|
||||
|
||||
let ldjson = doc.querySelector('script[type="application/ld+json"]')
|
||||
if (ldjson) {
|
||||
cleanRecipe(ldjson)
|
||||
} else {
|
||||
alert("No recipe found")
|
||||
}
|
||||
|
||||
|
||||
|
||||
function cleanRecipe(ldjson) {
|
||||
|
||||
try {
|
||||
var json = JSON.parse(ldjson.innerText); //JSON.parse(data);
|
||||
if (json["@type"] != "Recipe") {
|
||||
json = (json["@graph"] ?? json).find((item) => item["@type"] == "Recipe")
|
||||
let findLDJsonRecipe = (document) => {
|
||||
let lds = Array.from(document.querySelectorAll('script[type="application/ld+json"]'))
|
||||
.map(e => e.innerText)
|
||||
.sort((a,b) => a.length - b.length)
|
||||
|
||||
for (const ld of lds) {
|
||||
let r = JSON.parse(ld);
|
||||
if (r["@type"] != "Recipe") {
|
||||
r = Array.isArray(r) ? r : r["@graph"]
|
||||
r = r?.find((item)=>item["@type"]=="Recipe")
|
||||
}
|
||||
if (!r) continue;
|
||||
|
||||
delete json.review;
|
||||
delete json.video;
|
||||
|
||||
let f = new FileReader();
|
||||
f.onload = function(e) {
|
||||
parent.postMessage({replaceURL:e.target.result, compressURL:"true"}, "*");
|
||||
};
|
||||
f.readAsDataURL(new Blob([JSON.stringify(json)],{type : 'application/ld+json;charset=utf-8'}));
|
||||
|
||||
} catch (e) {
|
||||
console.debug("Data", e, {ldjson});
|
||||
delete r.review;
|
||||
delete r.video;
|
||||
if (!r.url) r.url = location.href;
|
||||
|
||||
console.log("Found Recipe:", r)
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
function findMicrodataRecipe(doc) {
|
||||
return parseMicrodata(doc).find((item)=>item["@type"]=="Recipe");
|
||||
}
|
||||
|
||||
let scrapeRecipe = async document => {
|
||||
|
||||
// Get title
|
||||
let recipe = {};
|
||||
recipe.name = document.title
|
||||
|
||||
// Get opengraph metadata
|
||||
let title = document.evaluate('//meta[@property="og:title"]/@content', document, null, XPathResult.STRING_TYPE)?.stringValue;
|
||||
if (title) recipe.name = title
|
||||
|
||||
let siteName = document.evaluate('//meta[@property="og:site_name"]/@content', document, null, XPathResult.STRING_TYPE)?.stringValue;
|
||||
if (siteName) recipe.publisher = {name:siteName}
|
||||
|
||||
recipe.image = document.evaluate('//meta[@property="og:image"]/@content', document, null, XPathResult.STRING_TYPE)?.stringValue;
|
||||
|
||||
// TODO: get siteLogo
|
||||
|
||||
// Get microformat metadata
|
||||
// let microName = document.evaluate('//*[@itemprop="name"]/@content', document, null, XPathResult.STRING_TYPE)?.stringValue;
|
||||
// if (microName) recipe.name = microName;
|
||||
|
||||
// recipe.recipeYield = document.evaluate('//*[@itemprop="recipeYield"]/text()', document, null, XPathResult.STRING_TYPE)?.stringValue;
|
||||
// recipe.totalTime = document.evaluate('//*[@itemprop="totalTime"]/text()', document, null, XPathResult.STRING_TYPE)?.stringValue;
|
||||
|
||||
// let ingredients = []
|
||||
// var xpath = `//*[@itemprop="recipeIngredient"]`;
|
||||
// const match = document.evaluate(xpath, document, null, XPathResult.ANY_TYPE, null);
|
||||
// let node = null;
|
||||
// while (node = match.iterateNext()) {
|
||||
// ingredients.push(node.innerText)
|
||||
// }
|
||||
// if (ingredients.length) recipe.recipeIngredient = ingredients;
|
||||
|
||||
// {
|
||||
// var xpath = `//*[@itemprop="recipeInstructions"]`;
|
||||
// const match = document.evaluate(xpath, document, null, XPathResult.ANY_TYPE, null);
|
||||
// let node = null;
|
||||
// let instructions = [];
|
||||
// while (node = match.iterateNext()) {
|
||||
// console.warn("match", node.innerText)
|
||||
// instructions.push(node.innerText);
|
||||
// }
|
||||
|
||||
// if (instructions.length == 1) instructions = instructions[0].trim().split("\n").filter((a)=>a.length > 1)
|
||||
// recipe.recipeInstructions = instructions;
|
||||
// }
|
||||
|
||||
|
||||
|
||||
// Scrape Yield
|
||||
if (!recipe.recipeYield) {
|
||||
for(let text in ["yield", "makes", "serv"]) {
|
||||
var xpath = `//div[starts-with(text(), '${text}')]`;
|
||||
const match = document.evaluate(xpath, document, null, XPathResult.ANY_TYPE, null);
|
||||
let node = null;
|
||||
while (node = match.iterateNext()) {
|
||||
recipe.recipeYield = node.innerText
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Scrape ingredients
|
||||
if (!recipe.recipeIngredient) {
|
||||
var xpath = "//div[text()='Ingredients']";
|
||||
const match = document.evaluate(xpath, document, null, XPathResult.ANY_TYPE, null);
|
||||
let node = null;
|
||||
while (node = match.iterateNext()) {
|
||||
console.warn("match", node.parentNode.innerText.split("\n"))
|
||||
recipe.recipeIngredient = node.parentNode.innerText.trim().split("\n")
|
||||
}
|
||||
}
|
||||
|
||||
// Scrape instructions
|
||||
if (!recipe.recipeInstructions) {
|
||||
var xpath = "//div[text()='Instructions']";
|
||||
const match = document.evaluate(xpath, document, null, XPathResult.ANY_TYPE, null);
|
||||
let node = null;
|
||||
while (node = match.iterateNext()) {
|
||||
console.warn("Instructions", node.parentNode.innerText.split("\n"))
|
||||
recipe.recipeInstructions = node.parentNode.innerText.trim().split("\n").filter((a)=>a.length > 1)
|
||||
}
|
||||
}
|
||||
|
||||
console.warn("Extracted", JSON.stringify(recipe,undefined,2))
|
||||
|
||||
// return undefined
|
||||
return recipe;
|
||||
}
|
||||
|
||||
|
||||
|
||||
let recipe = findLDJsonRecipe(doc) || findMicrodataRecipe(doc) || await scrapeRecipe(doc);
|
||||
if (recipe) {
|
||||
|
||||
let url = doc.evaluate('//meta[@property="og:url"]/@content', doc, null, XPathResult.STRING_TYPE)?.stringValue;
|
||||
if (url) recipe.mainEntityOfPage = url
|
||||
|
||||
let f = new FileReader();
|
||||
f.onload = function(e) {
|
||||
parent.postMessage({replaceURL:e.target.result, compressURL:"true"}, "*");
|
||||
};
|
||||
f.readAsDataURL(new Blob([JSON.stringify(recipe)],{type : 'application/ld+json;charset=utf-8'}));
|
||||
} else {
|
||||
parent.postMessage({title:"No Recipe Found...", error:"No recipe was found on this page."}, "*");
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
function parseMicrodata(doc) {
|
||||
let parsedElements = []
|
||||
const microdata = [];
|
||||
|
||||
let sanitize = (input) => input?.replace(/\s/gi, ' ').trim();
|
||||
|
||||
function addValue(obj, key, value) {
|
||||
if (Array.isArray(obj[key])) {
|
||||
obj[key].push(value);
|
||||
} else if (obj[key]) {
|
||||
obj[key] = [obj[key], value];
|
||||
} else {
|
||||
obj[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
function parseItem(elem) {
|
||||
const data = { }
|
||||
data["@type"] = elem.getAttribute('itemtype')?.split("/").pop();
|
||||
if (elem.getAttribute('itemid')) data["@id"] = elem.getAttribute('itemid');
|
||||
traverseItem(elem, data);
|
||||
parsedElements.push(elem);
|
||||
return data;
|
||||
}
|
||||
|
||||
function traverseItem(item, data) {
|
||||
for (const child of item.children) {
|
||||
if (!child.hasAttribute('itemprop')) {
|
||||
traverseItem(child, data);
|
||||
continue;
|
||||
}
|
||||
|
||||
const itemProps = child.getAttribute('itemprop').split(' ');
|
||||
if (child.hasAttribute('itemscope')) {
|
||||
const childData = parseItem(child)
|
||||
itemProps.forEach(key => addValue(data, key, childData));
|
||||
} else {
|
||||
itemProps.forEach(key => {
|
||||
let value = key === 'url' ? child.href : sanitize(child.content || child.textContent || child.src);
|
||||
addValue(data, key, value)
|
||||
});
|
||||
traverseItem(child, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
doc.querySelectorAll("[itemscope]").forEach(function(elem, i) {
|
||||
if (!parsedElements.includes(elem)) {
|
||||
microdata.push(parseItem(elem));
|
||||
}
|
||||
});
|
||||
console.log("micro", doc, microdata)
|
||||
return microdata;
|
||||
}
|
||||
|
|
@ -27,7 +27,7 @@ var isWatch = (window.outerWidth < 220);
|
|||
if (isWatch) document.documentElement.classList.add("watch")
|
||||
|
||||
let ignoredTerms = [
|
||||
"freshly", "use", "dry", "but", "broken", "pieces", "tbsp", "tsp", "peeled", "then", "can", "oz", "fresh", "out", "not", "sprig", "sprigs", "room", "temperature", "still", "see", "notes", "with", "beat", "together", "crust", "very", "cold", "hot", "top", "warm", "one", "note", "teaspoon", "teaspoons", "tablespoon", "tablespoons", "cup", "cups", "taste", "more", "melted", "into", "wide", "pound", "pounds", "gram", "grams", "you", "ounce", "ounces", "thinly", "sliced",
|
||||
"extra", "optional", "separate", "freshly", "use", "dry", "but", "broken", "pieces", "tbsp", "tsp", "peeled", "then", "can", "oz", "fresh", "out", "not", "sprig", "sprigs", "room", "temperature", "still", "see", "notes", "with", "beat", "together", "crust", "very", "cold", "hot", "top", "warm", "one", "note", "teaspoon", "teaspoons", "tablespoon", "tablespoons", "cup", "cups", "taste", "more", "melted", "into", "wide", "pound", "pounds", "gram", "grams", "you", "ounce", "ounces", "thinly", "sliced",
|
||||
"pan", "cube", "cubes", "finely", "ground", "garnish", "about", "cut", "and", "smashed", "each", "the", "medium", "large", "small", "for", "chopped", "minced", "grated", "box", "softened", "directed", "shredded", "cooked", "from", "frozen", "thawed"
|
||||
]
|
||||
let emojiMap = {
|
||||
|
|
@ -61,6 +61,8 @@ let emojiMap = {
|
|||
"carrot": "🥕",
|
||||
"corn": "🌽",
|
||||
"spicy": "🌶️",
|
||||
"chard": "🥬",
|
||||
"peppers": "🫑",
|
||||
"bell pepper": "🫑",
|
||||
"cucumber": "🥒",
|
||||
"leafy green": "🥬",
|
||||
|
|
@ -424,12 +426,16 @@ function faviconForTitle(title) {
|
|||
|
||||
function render() {
|
||||
try {
|
||||
let data = JSON.parse(window.params.body);
|
||||
var json = data; //JSON.parse(data);
|
||||
if (!json["@type"]?.includes("Recipe")) {
|
||||
json = (json["@graph"] ?? json).find((item) => item["@type"] == "Recipe")
|
||||
let r = JSON.parse(window.params.body);
|
||||
|
||||
if (r["@type"] != "Recipe") {
|
||||
r = Array.isArray(r) ? r : r["@graph"]
|
||||
r = r?.find((item)=>item["@type"]=="Recipe")
|
||||
}
|
||||
console.log("🍗 Rendering Recipe:", json);
|
||||
|
||||
var json = r || JSON.parse(window.params.body)
|
||||
console.log("🍗 Rendering Recipe:", json, JSON.parse(window.params.body));
|
||||
|
||||
} catch (e) {
|
||||
|
||||
document.body.appendChild(m("div", "Error parsing recipe", m("p", e.message), m("p", e.stack), m("pre", window.params.body) ));
|
||||
|
|
@ -445,9 +451,10 @@ function render() {
|
|||
|
||||
if (!json) return;
|
||||
let image = json.image;
|
||||
if (Array.isArray(image)) image = image.shift();
|
||||
if (Array.isArray(image)) image = image.pop();
|
||||
image = image?.url || image;
|
||||
let instructions = json.recipeInstructions;
|
||||
let instructions = json.recipeInstructions || [];
|
||||
console.log(json.name)
|
||||
let title = clean(json.name);
|
||||
let description = json.description || json.articleBody;
|
||||
description = clean(description?.replace(/\\n/g, "<br>"))
|
||||
|
|
@ -460,6 +467,7 @@ function render() {
|
|||
var ingredientTerms = new Set(
|
||||
Array.from(ingredients.join("\n").matchAll(/(\p{L}\p{M}?)+/gu)).map(m => m[0].length > 2 ? m[0].toLowerCase(): "")
|
||||
);
|
||||
|
||||
if (typeof instructions === "string") instructions = [instructions]
|
||||
instructions = flattenInstructions(instructions)
|
||||
|
||||
|
|
@ -628,7 +636,7 @@ function render() {
|
|||
m("div.headercolumns",
|
||||
m("header",
|
||||
m("a.publisherlink", {href:originalURL, target:"_blank"},
|
||||
publisherImage ? m("img.publisher", { src: publisherImage }) : hostname,
|
||||
publisherImage ? m("img.publisher", { src: publisherImage }) : (json.publisher?.name || hostname),
|
||||
),
|
||||
m(".headerflex",
|
||||
m(".headerleft",
|
||||
|
|
@ -659,7 +667,7 @@ function render() {
|
|||
),
|
||||
m(".actions.print-hide.watch-hide",
|
||||
originalURL ? m("a.action", { title:"Open original", href: originalURL, target:"_blank"}, m(".icon.public", {innerHTML:icons.public})) : null,
|
||||
m("a.action", { title:"Share", onclick: share}, m(".icon.share", {innerHTML:icons.share})),
|
||||
navigator.share ? m("a.action", { title:"Share", onclick: share}, m(".icon.share", {innerHTML:icons.share})) : undefined,
|
||||
// m("a.action", { title:"Show steps as list", onclick: () => {reformat = !reformat; render(); return false;}}, m(".icon.checklist")),
|
||||
m("a.action", { title:"Print", onclick: () => {window.print(); return false;} }, m(".icon.print", {innerHTML:icons.print})),
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue