{
  "title": "Deep Crawling",
  "video_id": "5bcdfsNkaVs",
  "published": "2024-05-16T11:27:11+0000",
  "complexity": "Normal",
  "old_url": "https://workflows-of-ai.com/workflow/deep-crawling",
  "description": "Crawl a full website with the Simple Crawler depth crawler in Drupal and use every page as context for answers from OpenAI.",
  "subtitle": "Crawl a whole documentation site with Simple Crawler and let OpenAI answer questions from all of its pages.",
  "summary": "The Simple Crawler depth crawler fetches the Guzzle documentation two levels deep and stores each page in an unlimited text field. A GPT-4o prompt in advanced mode then answers a question from all pages at once and references the source URL. Simple Crawler and ScrapingBot are compared, and Marcus talks about joining FreelyGive.",
  "categories": [
    "Drupal AI"
  ],
  "intro_html": "<p>Sometimes one webpage is not enough context to answer the questions you might have, or you want to fill some database/Search API AI with a lot of context for your chatbot.</p><p>Now it is possible using the Simple Crawler or the ScrapingBot in Drupal. They now have depth crawlers that go and scrape links following whatever rules you set and store it all as context.</p><p>In this example we use the context to answer some questions using OpenAI.</p><p>Modules used:<br><a href=\"https://www.drupal.org/project/ai_interpolator_openai\">https://www.drupal.org/project/ai_interpolator_openai</a><br><a href=\"https://www.drupal.org/project/ai_interpolator_simple_crawler\">https://www.drupal.org/project/ai_interpolator_simple_crawler</a></p>",
  "intro_added": false,
  "chapters": [
    "0:00 Crawling docs to answer questions",
    "1:33 Joining FreelyGive",
    "7:20 Scraping protection and Guzzle docs",
    "8:08 Why an LLM needs context",
    "10:59 Simple Crawler or ScrapingBot",
    "14:49 Simple Crawler options",
    "16:32 The Guzzle answering machine",
    "18:58 New depth crawler rules",
    "21:04 Depth crawler settings",
    "24:42 Article segmentation crawler mode",
    "26:38 Answer field in advanced mode",
    "31:42 Asking how to access headers",
    "34:24 Tuning the prompt and model"
  ],
  "corrections": [
    [
      "AI inator",
      "AI Interpolator"
    ],
    [
      "AI interpolator",
      "AI Interpolator"
    ],
    [
      "freely give",
      "FreelyGive"
    ],
    [
      "druple",
      "Drupal"
    ],
    [
      "drupa was",
      "Drupal was"
    ],
    [
      "Lang chain",
      "LangChain"
    ],
    [
      "Jango",
      "Django"
    ],
    [
      "guzzle",
      "Guzzle"
    ],
    [
      "guzel",
      "Guzzle"
    ],
    [
      "gusle",
      "Guzzle"
    ],
    [
      "gasel",
      "Guzzle"
    ],
    [
      "GLE doesn't",
      "Guzzle doesn't"
    ],
    [
      "this GLE documentation",
      "this Guzzle documentation"
    ],
    [
      "docs. Cle php. org",
      "docs.guzzlephp.org"
    ],
    [
      "open AI",
      "OpenAI"
    ],
    [
      "GPT 40",
      "GPT-4o"
    ],
    [
      "scraping bot",
      "ScrapingBot"
    ],
    [
      "use scraping butt",
      "use ScrapingBot"
    ],
    [
      "Milla Foundation",
      "Mozilla Foundation"
    ],
    [
      "on Rush",
      "on Drush"
    ]
  ],
  "ai_note": "The video and the introduction are Marcus's own, first published on workflows-of-ai.com. AI wrote the subtitle, the summary and the video chapters from the transcript, and corrected misheard names in YouTube's automatic captions."
}