Website Content Crawler

Website Content Crawler

Website Content Crawler : scrap any website content with meta title and meta description and all sites link. this tool will be useful for anylysis any website content and can use for llm model for ai training purpose.

#how to use

-- insert any website url -- insert deep level like how much deep scraping ..default is 0 level

output response like this

Result Data

[
  {
    "url": "https://www.apify.com/",
    "title": "Apify: The largest marketplace of trusted tools for AI",
    "meta_description": "Thousands of tools to automate your business. Get real-time web data, track competitors, generate leads, and integrate your apps and AI agents.",
    "content :""
    "emails": [],
    "links": [
      "#main",
      "/",
      "https://console.apify.com/sign-up",
      "https://console.apify.com/sign-in",
      "/store",
      "/store",
      "/actors",
      "/integrations",
      "https://mcp.apify.com/",
      "/anti-blocking",
      "/proxy",
      "https://crawlee.dev/",
      "https://mcp.apify.com/",
      "/enterprise",
      "/resources/startups",
      "/resources/universities",
      "/resources/nonprofits",
      "/use-cases/data-for-generative-ai",
      "/use-cases/data-for-ai-agents",
      "/use-cases/lead-generation",
      "/use-cases/market-research",
      "/use-cases",
      "/professional-services",
      "/partners",
      "https://docs.apify.com/",
      "/templates",
      "https://docs.apify.com/academy",
      "/partners/actor-developers",
      "https://docs.apify.com/api",
      "https://docs.apify.com/cli/",
      "https://docs.apify.com/sdk",
      "https://docs.apify.com/platform/integrations/mcp",
      "https://crawlee.dev/",
      "/partners/actor-developers",
      "https://help.apify.com/en/",
      "/ideas",
      "/change-log",
      "/success-stories",
      "/about",
      "/contact",
      "https://blog.apify.com/",
      "https://lu.ma/apify",
      "/partners",
      "/jobs",
      "https://discord.com/invite/jyEM2PRvMU",
      "/pricing",
      "/contact-sales",
      "/store/collections/mcp-connectors",
      "/store/collections/mcp-connectors",
      "/clockworks/tiktok-scraper",
      "/compass/crawler-google-places",
      "/apify/instagram-scraper",
      "/apify/website-content-crawler",
      "/apify/e-commerce-scraping-tool",
      "/apify/facebook-posts-scraper",
      "/clockworks/tiktok-scraper",
      "/compass/crawler-google-places",
      "/apify/instagram-scraper",
      "/apify/website-content-crawler",
      "/apify/e-commerce-scraping-tool",
      "/apify/facebook-posts-scraper",
      "/store",
      "/store",
      "/actors",
      "/professional-services",
      "https://blog.apify.com/announcing-mcp-connectors/",
      "https://crawlee.dev",
      "/templates/python-llamaindex-agent",
      "/templates/js-langchain",
      "/templates/ts-crawlee-playwright-chrome",
      "/templates/ts-crawlee-puppeteer-chrome",
      "/templates/ts-crawlee-cheerio",
      "/templates/python-selenium",
      "/templates/python-scrapy",
      "/templates/python-crawlee-beautifulsoup",
      "https://docs.apify.com/academy",
      "/templates",
      "https://discord.gg/w3e2v7rWDw",
      "/partners/actor-developers",
      "/contact-sales",
      "/enterprise",
      "https://blog.apify.com/intercom-customer-support-ai-chatbot-web-scraping",
      "https://blog.apify.com/groupon-reaches-new-merchants-with-web-data-collection",
      "https://blog.apify.com/how-web-scraping-ai-and-the-eu-have-come-together-to-sweep-away-fake-discounts-in-europe",
      "/success-stories",
      "/integrations",
      "https://docs.apify.com/api",
      "https://console.apify.com/sign-up",
      "/contact-sales/demo",
      "/store",
      "/integrations",
      "/proxy",
      "https://mcp.apify.com/",
      "https://crawlee.dev/",
      "https://docs.apify.com/",
      "/templates",
      "https://docs.apify.com/api",
      "/partners/actor-developers",
      "/professional-services",
      "/partners",
      "https://help.apify.com/en/",
      "/ideas",
      "https://discord.apify.com/",
      "/api",
      "https://blog.apify.com/what-is-web-scraping/",
      "https://blog.apify.com/best-web-scraping-tools/",
      "https://blog.apify.com/what-are-the-best-python-web-scraping-libraries/",
      "/scrapers",
      "/about",
      "/contact",
      "https://lu.ma/apify",
      "https://blog.apify.com/",
      "/jobs",
      "/resources/brand",
      "/",
      "http://linkedin.com/company/apify/",
      "https://x.com/apify",
      "https://github.com/apify",
      "https://www.youtube.com/apify",
      "https://discord.com/invite/jyEM2PRvMU",
      "https://www.tiktok.com/@apifytech",
      "https://docs.apify.com/legal/gdpr-information",
      "https://trust.apify.com/",
      "https://www.getapp.com/business-intelligence-analytics-software/a/apify/",
      "https://www.softwareadvice.com/data-extraction/apify-profile/",
      "https://www.capterra.com/p/150854/Apify/",
      "https://www.g2.com/products/apify/reviews",
      "https://www.trustradius.com/products/apify/reviews",
      "https://crozdesk.com/software/apify",
      "https://status.apify.com/",
      "https://docs.apify.com/legal",
      "https://docs.apify.com/legal/general-terms-and-conditions",
      "https://docs.apify.com/legal/privacy-policy",
      "https://docs.apify.com/legal/cookie-policy",
      "https://docs.apify.com/legal"
    ]
  },