# Website Sitemaps Sitemap: https://www.rosarymart.com/en_sitemap.xml ################################################### # Apple Bot # URL: # Purpose: Apple clawler # Impact: Heavy Load # Action: slow down User-agent: Applebot/0.1 Crawl-delay: 8 ################################################### # Apple Bot # URL: # Purpose: Apple clawler # Impact: Heavy Load # Action: slow down User-agent: applebot Crawl-delay: 8 ################################################### # Bing crawler # URL: http://www.bing.com/bingbot.htm # Purpose: Bing search # Impact: Very heavy load on the site # Action: slow down User-agent: bingbot/2.0 Crawl-delay: 10 ################################################### # Bing crawler # URL: http://search.msn.com/msnbot.htm # Purpose: MSN search # Impact: heavy load on the site # Action: slow down User-agent: msnbot Crawl-delay: 8 ################################################### # Pinterest crawler # URL: http://www.pinterest.com/bot.html # Purpose: Pinterest crawler # Impact: heavy load on the site # Action: slow down User-agent: Pinterestbot/1.0 Disallow: / ################################################### # opensiteexplorer crawler # URL: http://www.opensiteexplorer.org/dotbot # Purpose: SEO tool # Impact: Puts medium load on the site # Action: Disallow from indexing site User-agent: dotbot Disallow: / ################################################### # megindex crawler # URL: https://megaindex.com/crawler # Purpose: SEO tool # Impact: Puts medium load on the site # Action: Disallow from indexing site User-agent: MegaIndex.ru Disallow: / User-agent: megaindex.com Disallow: / ################################################### # GrapeshotCrawler crawler # URL: http://www.grapeshot.co.uk # Purpose: SEO tool # Impact: Puts medium load on the site # Action: Disallow from indexing site User-agent: GrapeshotCrawler Disallow: / ################################################### # Serpstatbot crawler # URL: http://serpstatbot.com # Purpose: SEO advanced backlink tracking bot # Impact: Puts medium load on the site # Action: Disallow from indexing site User-agent: serpstatbot Disallow: / ################################################### # Curious George crawler # URL: www.analyticsseo.com/crawler # Purpose: SEO services UK # Impact: Puts medium load on the site # Action: Disallow from indexing site User-agent: Curious George Disallow: / ################################################### # adscanner SEO scanner # URL: http://seoscanners.net; # Purpose: SEO services # Impact: # Action: Disallow from indexing site User-agent: seoscanners.net Disallow: / ################################################### # adscanner Huawei International Pte Ltd # URL: # Purpose: SEO services # Impact: heavy load on the site # Action: Disallow from indexing site User-agent: LieBaoFast Disallow: / ################################################### # AspiegelBot same Ips as Huawei # URL: # Purpose: Spam Scan # Impact: Very heavy load on the site # Action: Disallow from indexing site User-agent: AspiegelBot Disallow: / ################################################### # Amazon bot -aggressive # URL: (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) # Purpose: fast crawling from Amazonbot/0.1; # Impact: Very heavy load on the site # Action: Disallow from indexing site User-agent: Amazonbot/0.1 User-agent: Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot Disallow: / ################################################### # Crawlers Setup User-agent: * User-agent: Googlebot-Image Disallow: # Allowable Index # Allow: /sitemap/ Allow: /*where/p/ Allow: /*where/limit/all.html Allow:/skin/frontend/ultimo/customtheme/images/logo.png Allow:/skin/frontend/ultimo/customtheme/images/logo_rm.gif Allow:/skin/frontend/ultimo/customtheme/images/rosarymart_logo.png # Directories Disallow: /404/ Disallow: /app/ Disallow: /cgi-bin/ Disallow: /dev/ Disallow: /downloader/ Disallow: /errors/ Disallow: /includes/ Disallow: /lib/ Disallow: /order_editor/ Disallow: /magento/ #Disallow: /media/ Disallow: /media/catalog/product/cache Disallow: /media/catalog/product/placeholder #Disallow: /media/catalog/product/ Disallow: /media/captcha/ Disallow: /media/dhl/ Disallow: /media/customer/ Disallow: /media/downladable/ Disallow: /media/import/ Disallow: /media/sales/ Disallow: /media/temp/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /scripts/ Disallow: /shell/ Disallow: /skin/ Disallow: /src/ Disallow: /stats/ Disallow: /var/ Disallow: /b2b/ Disallow: /dship/ # Paths (clean URLs) Disallow: /index.php/ Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ Disallow: /catalogsearch/ Disallow: /catalog/product/view/ Disallow: /search/ #Disallow: /catalog/product/gallery/ Disallow: /checkout/ Disallow: /checkout/cart/ Disallow: /onestepcheckout/ Disallow: /control/ Disallow: /contacts/ Disallow: /customer/ Disallow: /customize/ Disallow: /newsletter/ Disallow: /poll/ Disallow: /review/ Disallow: /sendfriend/ Disallow: /wishlist/ Disallow: /tag/ Disallow: /*where/ # Files Disallow: /api.php Disallow: /cron.php Disallow: /cron.sh Disallow: /scheduler_cron.sh Disallow: /error_log Disallow: /get.php Disallow: /install.php Disallow: /cleanup.php Disallow: /phpinfo.php Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /STATUS.txt Disallow: /README.txt Disallow: /RELEASE_NOTES.txt Disallow: /CVS # Paths (no clean URLs) Disallow: /*.php$ Disallow: /*?SID= Disallow: /*.svn$ Disallow: /*.idea$ Disallow: /*.sql$ Disallow: /*.tgz$ Disallow: /*.CVS Disallow: /*?limit=all Disallow: /*.Zip$ Disallow: /composer.json Disallow: /composer.lock # WORDPRESS DIRECTORIES & FILES Disallow: /blog/wp-content/upgrade/ Disallow: /blog/wp-admin/ Disallow: /blog/wp-includes/