Json+ld

README.md

@@ -48,7 +48,6 @@ recipeScraper("some.recipe.url").then(recipe => {
 - https://copykat.com/
 - https://damndelicious.net/
 - https://www.eatingwell.com/
-- https://www.epicurious.com/
 - https://www.food.com/
 - https://www.foodandwine.com/
 - https://www.foodnetwork.com/
@@ -78,7 +77,8 @@ recipeScraper("some.recipe.url").then(recipe => {
 - https://www.yummly.com/
 - https://www.jamieoliver.com/
 
-Don't see a website you'd like to scrape? Open an [issue](https://github.com/jadkins89/Recipe-Scraper/issues) and we'll do our best to add it.
+And many more! the list above is old fashioned scraping, but for all those websites who have google recipe ld json included, it will also work.
+
 
 ## Recipe Object
 
@@ -105,6 +105,8 @@ Depending on the recipe, certain fields may be left blank. All fields are repres
 
 ## Error Handling
 
+If a recipe is not found on the given url, the basic page info will be returned: title, image & description.
+
 If the url provided is invalid and a domain is unable to be parsed, an error message will be returned.
 
 ```javascript
@@ -114,24 +116,6 @@ recipeScraper("keyboard kitty").catch(error => {
 });
 ```
 
-If the url provided doesn't match a supported domain, an error message will be returned.
-
-```javascript
-recipeScraper("some.invalid.url").catch(error => {
-  console.log(error.message);
-  // => "Site not yet supported"
-});
-```
-
-If a recipe is not found on a supported domain site, an error message will be returned.
-
-```javascript
-recipeScraper("some.no.recipe.url").catch(error => {
-  console.log(error.message);
-  // => "No recipe found on page"
-});
-```
-
 If a page does not exist or some other 400+ error occurs when fetching, an error message will be returned.
 
 ```javascript
@@ -150,6 +134,8 @@ recipeScraper("some.improper.url").catch(error => {
 });
 ```
 
+
+
 ## Bugs
 
 With web scraping comes a reliance on the website being used not changing format. If this occurs we need to update our scrape. Please reach out if you are experiencing an issue.

helpers/BaseScraper.js

@@ -22,7 +22,7 @@ class BaseScraper {
 
       return res.ok; // res.status >= 200 && res.status < 300
     } catch (e) {
-      console.log(e)
+      // console.log(e)
       return false;
     }
   }
@@ -47,6 +47,192 @@ class BaseScraper {
     throw new Error("No recipe found on page");
   }
 
+  /**
+   * look for LD+JOSN script in the web page.
+   * @param {object} $ - a cheerio object representing a DOM
+   * @returns {boolean} - if exist, set recipe data and return true, else - return false.
+   */
+  defaultLD_JOSN($) {
+    const jsonLDs = Object.values($("script[type='application/ld+json']"));
+    let isRecipeSchemaFound = false;
+
+    jsonLDs.forEach(jsonLD => {
+      if (jsonLD && jsonLD.children && Array.isArray(jsonLD.children)) {
+        jsonLD.children.forEach(el => {
+          if (el.data) {
+
+            const jsonRaw = el.data;
+            const result = JSON.parse(jsonRaw);
+            let recipe;
+
+
+
+            if (result['@graph'] && Array.isArray(result['@graph'])) {
+              result['@graph'].forEach(g => {
+                if (g['@type'] === 'Recipe') {
+                  recipe = g;
+                }
+              })
+            }
+
+            if (result['@type'] === 'Recipe') {
+              recipe = result;
+            }
+
+            if (Array.isArray(result['@type']) && result['@type'].includes('Recipe')) {
+              recipe = result;
+            }
+
+            if (recipe) {
+              // console.log('found a Recipe type json schema!');
+              try {
+                // name
+                this.recipe.name = BaseScraper.HtmlDecode($, recipe.name);
+
+                // description
+                if (recipe.description) {
+                  this.recipe.description = BaseScraper.HtmlDecode($, recipe.description);
+                } else {
+                  this.defaultSetDescription($);
+                }
+
+                // image
+                if (Array.isArray(recipe.image)) {
+                  recipe.image = recipe.image[0];
+                }
+
+                if (recipe.image) {
+                  if (recipe.image["@type"] === "ImageObject" && recipe.image.url) {
+                    this.recipe.image = recipe.image.url;
+                  } else if (typeof recipe.image === "string") {
+                    this.recipe.image = recipe.image;
+                  }
+                } else {
+                  this.defaultSetImage($);
+                }
+
+
+                // tags
+                this.recipe.tags = [];
+                if (recipe.keywords) {
+                  if (typeof recipe.keywords === "string") {
+                    this.recipe.tags = [...recipe.keywords.split(',')]
+                  } else if (Array.isArray(recipe.keywords)) {
+                    this.recipe.tags = [...recipe.keywords]
+                  }
+                }
+
+                if (recipe.recipeCuisine) {
+                  if (typeof recipe.recipeCuisine === "string") {
+                    this.recipe.tags.push(recipe.recipeCuisine)
+                  } else if (Array.isArray(recipe.recipeCuisine)) {
+                    this.recipe.tags = [...new Set([...this.recipe.tags, ...recipe.recipeCuisine])]
+                  }
+                }
+
+                if (recipe.recipeCategory) {
+                  if (typeof recipe.recipeCategory === "string") {
+                    this.recipe.tags.push(recipe.recipeCategory)
+                  } else if (Array.isArray(recipe.recipeCategory)) {
+                    this.recipe.tags = [...new Set([...this.recipe.tags, ...recipe.recipeCategory])]
+                  }
+                }
+
+                this.recipe.tags = this.recipe.tags.map(i => BaseScraper.HtmlDecode($, i));
+                this.recipe.tags = [...new Set(this.recipe.tags)];
+
+                // ingredients
+                if (Array.isArray(recipe.recipeIngredient)) {
+                  this.recipe.ingredients = recipe.recipeIngredient.map(i => BaseScraper.HtmlDecode($, i));
+                } else if (typeof recipe.recipeIngredient === "string") {
+                  this.recipe.ingredients = recipe.recipeIngredient.split(",").map(i => BaseScraper.HtmlDecode($, i.trim()));
+                }
+
+                // instructions (may be string, array of strings, or object of sectioned instructions)
+                this.recipe.instructions = [];
+                this.recipe.sectionedInstructions = [];
+
+                if (recipe.recipeInstructions &&
+                  recipe.recipeInstructions["@type"] === "ItemList" &&
+                  recipe.recipeInstructions.itemListElement) {
+
+                  recipe.recipeInstructions.itemListElement.forEach(section => {
+                    this.recipe.instructions = [
+                      ...this.recipe.instructions,
+                      ...section.itemListElement.map(i => BaseScraper.HtmlDecode($, i.text))
+                    ];
+                    section.itemListElement.forEach(i => {
+                      this.recipe.sectionedInstructions.push({
+                        sectionTitle: section.name,
+                        text: BaseScraper.HtmlDecode($, i.text),
+                        image: i.image || ''
+                      })
+                    });
+                  });
+                } else if (Array.isArray(recipe.recipeInstructions)) {
+                  recipe.recipeInstructions.forEach(instructionStep => {
+                    if (instructionStep["@type"] === "HowToStep") {
+                      this.recipe.instructions.push(BaseScraper.HtmlDecode($, instructionStep.text));
+                      this.recipe.sectionedInstructions.push({
+                        sectionTitle: instructionStep.name || '',
+                        text: BaseScraper.HtmlDecode($, instructionStep.text),
+                        image: instructionStep.image || ''
+                      })
+                    } else if (instructionStep["@type"] === "HowToSection") {
+                      if (instructionStep.itemListElement) {
+                        instructionStep.itemListElement.forEach(step => {
+                          this.recipe.instructions.push(BaseScraper.HtmlDecode($, step.text));
+
+                          this.recipe.sectionedInstructions.push({
+                            sectionTitle: instructionStep.name,
+                            text: BaseScraper.HtmlDecode($, step.text),
+                            image: step.image || ''
+                          })
+                        });
+                      }
+                    } else if (typeof instructionStep === "string") {
+                      this.recipe.instructions.push(BaseScraper.HtmlDecode($, instructionStep));
+                    }
+                  });
+                } else if (typeof recipe.recipeInstructions === "string") {
+                  this.recipe.instructions = [BaseScraper.HtmlDecode($, recipe.recipeInstructions)]
+                }
+
+                // prep time
+                if (recipe.prepTime) {
+                  this.recipe.time.prep = BaseScraper.parsePTTime(recipe.prepTime);
+                }
+
+                // cook time
+                if (recipe.cookTime) {
+                  this.recipe.time.cook = BaseScraper.parsePTTime(recipe.cookTime);
+                }
+
+                // total time
+                if (recipe.totalTime) {
+                  this.recipe.time.total = BaseScraper.parsePTTime(recipe.totalTime);
+                }
+
+                // servings
+                if (Array.isArray(recipe.recipeYield)) {
+                  this.recipe.servings = recipe.recipeYield[0];
+                } else if (typeof recipe.recipeYield === "string") {
+                  this.recipe.servings = recipe.recipeYield;
+                }
+
+                isRecipeSchemaFound = true;
+              } catch (e) {
+                console.log(e);
+              }
+            }
+          }
+        });
+      }
+    });
+
+    return isRecipeSchemaFound;
+  }
+
   /**
    * @param {object} $ - a cheerio object representing a DOM
    * @returns {string|null} - if found, an image url
@@ -58,6 +244,21 @@ class BaseScraper {
       $("meta[itemprop='image']").attr("content");
   }
 
+  /**
+   * @param {object} $ - a cheerio object representing a DOM
+   * if found, set recipe name
+   */
+  defaultSetName($) {
+    let title =
+      $("meta[name='title']").attr("content") ||
+      $("meta[property='og:title']").attr("content") ||
+      $("meta[name='twitter:title']").attr("content");
+
+    title = title.split('|')[0];
+
+    this.recipe.name = title ? title.trim() : '';
+  }
+
   /**
    * @param {object} $ - a cheerio object representing a DOM
    * if found, set recipe description
@@ -77,11 +278,16 @@ class BaseScraper {
    */
   async fetchDOMModel() {
     try {
-      const res = await fetch(this.url);
+      const meta = [
+        ['User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15'],
+      ];
+      const headers = new fetch.Headers(meta);
+      const res = await fetch(this.url, {headers});
       const html = await res.text();
       return cheerio.load(html);
     } catch (err) {
-      this.defaultError();
+      throw err;
+      // this.defaultError();
     }
   }
 
@@ -91,9 +297,15 @@ class BaseScraper {
    */
   async fetchRecipe() {
     this.checkUrl();
-    const $ = await this.fetchDOMModel();
-    this.createRecipeObject();
-    this.scrape($);
+    try {
+      const $ = await this.fetchDOMModel();
+      this.createRecipeObject();
+      this.scrape($);
+    } catch (e) {
+      // throw e;
+      this.defaultError();
+    }
+
     return this.validateRecipe();
   }
 
@@ -110,25 +322,37 @@ class BaseScraper {
     return el.text().trim();
   }
 
+  static HtmlDecode($, s) {
+    const res = $('<div>').html(s).text() || "";
+
+    return res.trim()
+      .replace(/amp;/gm, '')
+      .replace(/(?=\[caption).*?(?<=\[ caption\])/g, '') // removes short-codes [caption.*[ caption]
+      .replace(/\n/g, "");
+  }
+
   /**
    * Validates scraped recipes against defined recipe schema
    * @returns {object} - an object representing the recipe
    */
   validateRecipe() {
     let res = validate(this.recipe, recipeSchema);
     if (!res.valid) {
+      // res.errors.forEach(error => {
+      //   console.log(error.property + ' ' + error.message);
+      // });
       this.defaultError();
     }
     return this.recipe;
   }
 
   static parsePTTime(ptTime) {
     ptTime = ptTime.replace('PT', '');
-    ptTime = ptTime.replace('H', ' hours');
-    ptTime = ptTime.replace('M', ' minutes');
+    ptTime = ptTime.replace('H', ' hours ');
+    ptTime = ptTime.replace('M', ' minutes ');
     ptTime = ptTime.replace('S', ' seconds');
 
-    return ptTime;
+    return ptTime.trim();
   }
 }
 

helpers/DefaultLdJsonScraper.js

@@ -0,0 +1,31 @@
+const BaseScraper = require("./BaseScraper");
+
+class DefaultLdJsonScraper extends BaseScraper {
+
+  // async customPoll(page) {
+  //   let container,
+  //     count = 0;
+  //   do {
+  //     container = await page.$("script[type='application/ld+json']");
+  //     if (!container) {
+  //       await page.waitForTimeout(100);
+  //       count++;
+  //     }
+  //   } while (!container && count < 60);
+  //   return true;
+  // }
+
+  scrape($) {
+    const isSchemaFound = this.defaultLD_JOSN($);
+
+    if (!isSchemaFound) {
+      // throw new Error("Site not yet supported");
+      // if no recipe schema was found, return the basic page info
+      this.defaultSetName($);
+      this.defaultSetDescription($);
+      this.defaultSetImage($);
+    }
+  }
+}
+
+module.exports = DefaultLdJsonScraper;