web crawler

1 reply [Last post]
shashwat
Offline
Last seen: 1 year 31 weeks ago
Joined: 2009-02-18
bOt Points: 1160
Re: web crawler

Hello apeksha0701,

Check out the code below -

<?php

class Crawler {

  

  protected $markup = '';

  

  public function __construct($uri) {

    $this->markup = $this->getMarkup($uri);  

  }

  

  public function getMarkup($uri) {

    return file_get_contents($uri);   

  }


  public function get($type) {

    $method = "_get_{$type}";

    if (method_exists($this, $method)){

      return call_user_method($method, $this);

    }

  }

  

  protected function _get_images() {

    if (!empty($this->markup)){

      preg_match_all('/<img([^>]+)\/>/i', $this->markup, $images);         

      return !empty($images[1]) ? $images[1] : FALSE;

    }

  }

  

  protected function _get_links() {

    if (!empty($this->markup)){

      preg_match_all('/<a([^>]+)\>(.*?)\<\/a\>/i', $this->markup, $links);  

      return !empty($links[1]) ? $links[1] : FALSE;

    }

  }

}


/********************

Change the url_to_crawl variable below depending

upon which page you want to crawl :)

*********************/


$url_to_crawl='http://www.botskool.com';



$crawl = new Crawler($url_to_crawl);

$images = $crawl->get('images');

$links = $crawl->get('links');



/**************************

The code below is just for properly

displaying the data that has been collected

by the PHP web crawler.

***************************/


echo '

<style type="text/css">

<!--

.style1 {font-size: 16px}

-->

</style>


<div align="center">

  <h1>PHP WEB CRAWLER</h1>

  <p class="style1">(Provided by <a href="http://www.botskool.com">www.botskool.com</a>) </p>

  <p class="style1"><strong>Below are the details of links and images that have been extracted from '. $url_to_crawl.'</strong>    </p>

</div>

';


reset($images);


while (list($key, $val) = each($images))

  {

  echo "<strong>Image $key Details </strong><br /> $val<br />---------------------------<br />";

  }


reset($links);


while (list($key, $val) = each($links))

  {

  echo "<strong>Link $key Details </strong><br /> $val<br />---------------------------<br />";

  }

 

?>

Sample output of the above code - 

PHP WEB CRAWLER

(Provided by www.botskool.com)

Below are the details of links and images that have been extracted from http://www.botskool.com

Image 0 Details
src="http://www.botskool.com/sites/default/files/arthemia_logo.png" alt="bOtskOOl" width="230"
---------------------------
Image 1 Details
src="http://www.botskool.com/sites/default/files/images/LinkedIn-Google-Plus-Profile-Pic-01.png" alt="Linkedin Logo" title="Linkedin Logo" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 2 Details
src="http://www.botskool.com/sites/default/files/images/icon%20mail100x100.jpeg" alt="Outlook on gmail" title="Outlook on gmail" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 3 Details
src="http://www.botskool.com/sites/default/files/images/Search%20engine%20optimization%20SEO_0.jpg" alt="search engine optimization SEO" title="search engine optimization SEO" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 4 Details
src="http://www.botskool.com/sites/default/files/images/Apple-Mac-bash-shellshock.jpg" alt="Apple Mac bash shellshock linux" title="Apple Mac bash shellshock linux" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 5 Details
src="http://www.botskool.com/sites/default/files/images/heartbleed-bug.thumbnail.jpg" alt="The HeartBleed Bug" title="The HeartBleed Bug" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 6 Details
src="http://www.botskool.com/sites/default/files/images/WordPress-Automatic-Updates-fully-automatic.thumbnail.png" alt="How to Configure Automatic Updates in WordPress" title="How to Configure Automatic Updates in WordPress" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 7 Details
src="http://www.botskool.com/sites/default/files/images/how-download-and-run-android-emulator-command-line-windows-thumbnail.thumbnail.png" alt="How to download and run Android Emulator via Command Line on Windows" title="How to download and run Android Emulator via Command Line on Windows" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 8 Details
src="http://www.botskool.com/sites/default/files/images/how-download-and-install-bluestacks-app-player-windows-thumbnail_0.thumbnail.png" alt="How to download and install BlueStacks App Player on Windows" title="How to download and install BlueStacks App Player on Windows" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 9 Details
height="26" width="88" style="border:0" alt="" src="https://feeds.feedburner.com/~fc/bOtskOOl?bg=99CCFF&fg=444444&anim=1"
---------------------------
Image 10 Details
alt="Syndicate content" title="Syndicate content" width="32" height="32" src="http://www.botskool.com/sites/default/files/image/botskool_feed.png"
---------------------------
Image 11 Details
alt="Email subscription" title="Subscribe via email" width="32" height="32" src="http://www.botskool.com/sites/default/files/image/botskool_email.png"
---------------------------
Image 12 Details
src="http://www.botskool.com/sites/default/files/image/123.gif" alt="Like bOtskOOl Facebook page now!" width="125" height="70"
---------------------------
Image 13 Details
src="http://www.botskool.com/sites/default/files/images/LinkedIn-Google-Plus-Profile-Pic-01.png" alt="Linkedin Logo" title="Linkedin Logo" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 14 Details
src="http://www.botskool.com/sites/default/files/images/icon%20mail100x100.jpeg" alt="Outlook on gmail" title="Outlook on gmail" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 15 Details
src="http://www.botskool.com/sites/default/files/images/Search%20engine%20optimization%20SEO_0.jpg" alt="search engine optimization SEO" title="search engine optimization SEO" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 16 Details
src="http://www.botskool.com/sites/default/files/images/Apple-Mac-bash-shellshock.jpg" alt="Apple Mac bash shellshock linux" title="Apple Mac bash shellshock linux" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 17 Details
src="http://www.botskool.com/sites/default/files/images/heartbleed-bug.thumbnail.jpg" alt="The HeartBleed Bug" title="The HeartBleed Bug" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 18 Details
src="http://www.botskool.com/sites/default/files/images/WordPress-Automatic-Updates-fully-automatic.thumbnail.png" alt="How to Configure Automatic Updates in WordPress" title="How to Configure Automatic Updates in WordPress" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 19 Details
src="http://www.botskool.com/sites/default/files/images/how-download-and-run-android-emulator-command-line-windows-thumbnail.thumbnail.png" alt="How to download and run Android Emulator via Command Line on Windows" title="How to download and run Android Emulator via Command Line on Windows" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 20 Details
src="http://www.botskool.com/sites/default/files/images/how-download-and-install-bluestacks-app-player-windows-thumbnail_0.thumbnail.png" alt="How to download and install BlueStacks App Player on Windows" title="How to download and install BlueStacks App Player on Windows" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 21 Details
src="http://www.botskool.com/sites/default/files/images/how-create-and-render-listview-autodividers-web-page-using-jquery-mobile-thumbnail.thumbnail.png" alt="How to create and render listview with autodividers in a web page using jQuery Mobile" title="How to create and render listview with autodividers in a web page using jQuery Mobile" class="image image-thumbnail " width="100" height="100"
---------------------------
Image 22 Details
title="Do not copy content from the page. Plagiarism will be detected by Copyscape." border="0" alt="Page copy protected against web site content infringement by Copyscape" width="234" height="16" src="http://www.botskool.com/images/other/copyscape.gif"
---------------------------
Image 23 Details
alt="Creative Commons License" src="http://www.botskool.com/images/other/cc88x31.png"
---------------------------
Link 0 Details
href="http://www.botskool.com/user/register"
---------------------------
Link 1 Details
href="http://www.botskool.com/user/login?destination=frontpage"
---------------------------
Link 2 Details
href="http://www.botskool.com/" title="bOtskOOl"
---------------------------
Link 3 Details
href="http://www.botskool.com/geeks" id="dhtml_menu-1622"
---------------------------
Link 4 Details
href="http://www.botskool.com/" title="" id="dhtml_menu-1733" class="active"
---------------------------
Link 5 Details
href="http://www.botskool.com/tutorials" title="Go to our tutorial section" id="dhtml_menu-524"
---------------------------
Link 6 Details
href="http://www.botskool.com/programming-tutorials" title="Explore our computer programming section" id="dhtml_menu-560"
---------------------------
Link 7 Details
href="http://www.botskool.com/forum" title="Go to bOtskOOl fOrum" id="dhtml_menu-175"
---------------------------
Link 8 Details
href="http://www.botskool.com/utilities" title="Online Tools by bOtLabs" id="dhtml_menu-1225"
---------------------------
Link 9 Details
href="http://www.botskool.com/online-compiler" title="Test our Online C/C++ Compiler now! You need to have an account for accessing the compiler. Register for free now!" id="dhtml_menu-608"
---------------------------
Link 10 Details
href="http://www.botskool.com/online-html-compiler" title="FREE Online HTML Editor - Try it out now!" id="dhtml_menu-1095"
---------------------------
Link 11 Details
href="http://www.botskool.com/utilities/converter/html-php-converter" title="HTML to PHP Converter" id="dhtml_menu-1228"
---------------------------
Link 12 Details
href="http://www.w3analyzer.com" title="Get complete website information including website worth, daily income, pr, backlink, traffic detail, directory listing." id="dhtml_menu-1423"
---------------------------
Link 13 Details
href="http://www.botskool.com/utilities/online-image-processing/convolution-filter" title="Convolution Filter" id="dhtml_menu-1229"
---------------------------
Link 14 Details
href="http://phploginanydata.botskool.com" title="PHP Login Script" id="dhtml_menu-1442"
---------------------------
Link 15 Details
href="http://server-plus-plus.botskool.com/" title="An open source PHP based server monitoring script." id="dhtml_menu-1705"
---------------------------
Link 16 Details
href="https://chrome.google.com/extensions/detail/aijniobjengidecipjnbejabdhcpdakm" title="FC Barcelona Chrome Extension: A tribute to total football " id="dhtml_menu-1706"
---------------------------
Link 17 Details
href="http://www.botskool.com/user-pages" title="Contains tutorials and articles written by bOtskOOl Users" id="dhtml_menu-1098"
---------------------------
Link 18 Details
href="http://www.botskool.com/bot-fun" title="Play free online games!" id="dhtml_menu-548"
---------------------------
Link 19 Details
href="http://www.botskool.com/games/arcade-games/18wheeler/home" title="Play 18 wheeler now!" id="dhtml_menu-569"
---------------------------
Link 20 Details
href="http://www.botskool.com/games/racing-games/simtaxi4/home" title="Another day in the big city.. from a taxi drivers point of view!" id="dhtml_menu-570"
---------------------------
Link 21 Details
href="http://www.botskool.com/games/shooting-games/battleoverberlin/home" title="" id="dhtml_menu-571"
---------------------------
Link 22 Details
href="http://www.botskool.com/games/racing-games/smugglers/home" title="Drive your jeep, pick up cargo and deliver it, while avoiding the cops." id="dhtml_menu-572"
---------------------------
Link 23 Details
href="http://www.botskool.com/games/shooting-games/urbansniper/home" title="Find your mark and become the urban sniper!" id="dhtml_menu-573"
---------------------------
Link 24 Details
href="http://www.botskool.com/games/arcade-games/skyfirefighter/home" title="Fight with the forest fire!" id="dhtml_menu-575"
---------------------------
Link 25 Details
href="http://www.botskool.com/games/simulation-games/airport-madness-3" title="Airport Madness 3" id="dhtml_menu-1711"
---------------------------
Link 26 Details
href="http://www.botskool.com/games/puzzle-games/mouse-maze" title="Mouse Maze" id="dhtml_menu-1712"
---------------------------
Link 27 Details
href="http://www.botskool.com/tutorials"
---------------------------
Link 28 Details
href="http://www.botskool.com/tutorials"
---------------------------
Link 29 Details
href="http://www.botskool.com/online-compiler"
---------------------------
Link 30 Details
href="http://www.botskool.com/online-compiler"
---------------------------
Link 31 Details
href="http://www.botskool.com/category/tags/android"
---------------------------
Link 32 Details
href="http://www.botskool.com/category/tags/android"
---------------------------
Link 33 Details
href="http://www.botskool.com/forum"
---------------------------
Link 34 Details
href="http://www.botskool.com/forum"
---------------------------
Link 35 Details
href="http://www.botskool.com/programming-tutorials"
---------------------------
Link 36 Details
href="http://www.botskool.com/programming-tutorials"
---------------------------
Link 37 Details
class="twitter-share-button" data-count="horizontal" href="https://twitter.com/share"
---------------------------
Link 38 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500"
---------------------------
Link 39 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500"
---------------------------
Link 40 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500"
---------------------------
Link 41 Details
href="http://www.botskool.com/geeks/2014/11/how-configure-gmail-outlook"
---------------------------
Link 42 Details
href="http://www.botskool.com/geeks/2014/11/how-configure-gmail-outlook"
---------------------------
Link 43 Details
href="http://www.botskool.com/geeks/2014/11/how-configure-gmail-outlook"
---------------------------
Link 44 Details
href="http://www.botskool.com/geeks/best-search-engine-optimization-seo-techniques-improve-blog-traffic"
---------------------------
Link 45 Details
href="http://www.botskool.com/geeks/best-search-engine-optimization-seo-techniques-improve-blog-traffic"
---------------------------
Link 46 Details
href="http://www.botskool.com/geeks/best-search-engine-optimization-seo-techniques-improve-blog-traffic"
---------------------------
Link 47 Details
href="http://www.botskool.com/geeks/2014/09/how-protect-mac-or-linux-against-shellshock-exploit-aka-bash-vulnerability"
---------------------------
Link 48 Details
href="http://www.botskool.com/geeks/2014/09/how-protect-mac-or-linux-against-shellshock-exploit-aka-bash-vulnerability"
---------------------------
Link 49 Details
href="http://www.botskool.com/geeks/2014/05/heartbleed-bug" target="_blank"
---------------------------
Link 50 Details
href="http://www.botskool.com/geeks/2014/09/how-protect-mac-or-linux-against-shellshock-exploit-aka-bash-vulnerability"
---------------------------
Link 51 Details
href="http://www.botskool.com/geeks/2014/05/heartbleed-bug"
---------------------------
Link 52 Details
href="http://www.botskool.com/geeks/2014/05/heartbleed-bug"
---------------------------
Link 53 Details
href="http://www.botskool.com/geeks/2014/05/heartbleed-bug"
---------------------------
Link 54 Details
href="http://www.botskool.com/geeks/how-configure-automatic-updates-wordpress"
---------------------------
Link 55 Details
href="http://www.botskool.com/geeks/how-configure-automatic-updates-wordpress"
---------------------------
Link 56 Details
href="http://www.botskool.com/geeks/how-configure-automatic-updates-wordpress"
---------------------------
Link 57 Details
href="http://www.botskool.com/geeks/how-download-and-run-android-emulator-command-line-windows"
---------------------------
Link 58 Details
href="http://www.botskool.com/geeks/how-download-and-run-android-emulator-command-line-windows"
---------------------------
Link 59 Details
href="http://www.botskool.com/geeks/how-download-and-run-android-emulator-command-line-windows"
---------------------------
Link 60 Details
href="http://www.botskool.com/geeks/how-download-and-install-bluestacks-app-player-windows"
---------------------------
Link 61 Details
href="http://www.botskool.com/geeks/how-download-and-install-bluestacks-app-player-windows"
---------------------------
Link 62 Details
href="http://www.botskool.com/geeks/how-download-and-install-bluestacks-app-player-windows"
---------------------------
Link 63 Details
class="twitter-share-button" data-count="horizontal" href="https://twitter.com/share"
---------------------------
Link 64 Details
href="https://feeds.feedburner.com/bOtskOOl"
---------------------------
Link 65 Details
class="feed-icon" target="_blank" href="https://feeds.feedburner.com/bOtskOOl"
---------------------------
Link 66 Details
class="email-icon" target="_blank" href="https://feedburner.google.com/fb/a/mailverify?uri=bOtskOOl"
---------------------------
Link 67 Details
href="https://cloud.feedly.com/#subscription%2Ffeed%2Fhttp%3A%2F%2Fwww.botskool.com%2Frss.xml" target='blank'
---------------------------
Link 68 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3284"
---------------------------
Link 69 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3283"
---------------------------
Link 70 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3282"
---------------------------
Link 71 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3281"
---------------------------
Link 72 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3280"
---------------------------
Link 73 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3279"
---------------------------
Link 74 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3278"
---------------------------
Link 75 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3277"
---------------------------
Link 76 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3276"
---------------------------
Link 77 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500#comment-3275"
---------------------------
Link 78 Details
href="http://www.botskool.com/forum/robotics/robotics-contest/adidas-nmd-black-sale-2017" title="56 comments"
---------------------------
Link 79 Details
href="http://www.botskool.com/forum/robotics/general-robotics/candela-akrabor-porgan-and-kalan-bulgaria" title="419 comments"
---------------------------
Link 80 Details
href="http://www.botskool.com/forum/robotics/general-robotics/transfer-money-usa"
---------------------------
Link 81 Details
href="http://www.botskool.com/forum/robotics/general-robotics/tomasshurf"
---------------------------
Link 82 Details
href="http://www.botskool.com/forum/robotics/general-robotics/stewartrardy"
---------------------------
Link 83 Details
href="http://www.botskool.com/forum" title="Read the latest forum topics."
---------------------------
Link 84 Details
href="http://www.botskool.com/geeks/2015/03/how-check-number-connections-linkedin-if-we-exceed-500"
---------------------------
Link 85 Details
href="http://www.botskool.com/geeks/2014/11/how-configure-gmail-outlook"
---------------------------
Link 86 Details
href="http://www.botskool.com/geeks/best-search-engine-optimization-seo-techniques-improve-blog-traffic"
---------------------------
Link 87 Details
href="http://www.botskool.com/geeks/2014/09/how-protect-mac-or-linux-against-shellshock-exploit-aka-bash-vulnerability"
---------------------------
Link 88 Details
href="http://www.botskool.com/geeks/2014/05/heartbleed-bug"
---------------------------
Link 89 Details
href="http://www.botskool.com/geeks/how-configure-automatic-updates-wordpress"
---------------------------
Link 90 Details
href="http://www.botskool.com/geeks/how-download-and-run-android-emulator-command-line-windows"
---------------------------
Link 91 Details
href="http://www.botskool.com/geeks/how-download-and-install-bluestacks-app-player-windows"
---------------------------
Link 92 Details
href="http://www.botskool.com/geeks/how-create-and-render-listview-autodividers-web-page-using-jquery-mobile"
---------------------------
Link 93 Details
href="http://www.botskool.com/tutorials/mechanical/basic-mechanisms-robotics"
---------------------------
Link 94 Details
href="http://www.botskool.com/tutorials/electronics/general-electronics/motor-controller-h-bridge"
---------------------------
Link 95 Details
href="http://www.botskool.com/tutorials/fourth-year-projects/avr-line-follower"
---------------------------
Link 96 Details
href="http://www.botskool.com/tutorials/electronics/general-electronics/building-electronic-circuits-using-breadboard"
---------------------------
Link 97 Details
href="http://www.botskool.com/tutorials/electronics/8051/ram-memory-allocation-and-addressing-modes"
---------------------------
Link 98 Details
href="http://www.botskool.com/programming-tutorials/linux/install-linuxmint-sun-virtualbox-3-quick-steps-screenshots-step-1"
---------------------------
Link 99 Details
href="http://www.botskool.com/programming-tutorials/linux/filesystem"
---------------------------
Link 100 Details
href="http://www.botskool.com/programming-tutorials/c-examples/finding-factorial-number-using-recursion"
---------------------------
Link 101 Details
href="http://www.botskool.com/programming-tutorials/linux/myths"
---------------------------
Link 102 Details
href="http://www.botskool.com/programming-tutorials/c-examples/highest-common-factors-numbers"
---------------------------
Link 103 Details
href="http://www.copyscape.com/"
---------------------------
Link 104 Details
rel="license" href="https://creativecommons.org/licenses/by-nc-nd/2.5/in/"
---------------------------
Link 105 Details
rel="attributionURL" property="attributionName" cc="http://creativecommons.org/ns#" href="http://www.botskool.com"
---------------------------
Link 106 Details
rel="license" href="https://creativecommons.org/licenses/by-nc-nd/2.5/in/"
---------------------------
Link 107 Details
rel="morePermissions" cc="http://creativecommons.org/ns#" href="http://www.botskool.com/contact"
---------------------------
Link 108 Details
href="http://www.botskool.com/privacy-policy-botskool-com"
---------------------------
Link 109 Details
href="http://www.botskool.com/bot-xchange-product-listing-policy"
---------------------------
Link 110 Details
href="http://www.botskool.com/intellectual-property-rights-ipr-protection-policy"
---------------------------

Change accordingly the $url_to_crawl variable depending upon which page you wanna crawl 

Now you can store this data in some database such as MySQL and your web crawler is ready along with its own database backend. 

n/a