# robots.txt for https://www.wheretostay.com/

User-agent: *
Allow: /
Disallow: /reviews/		 	# post review
Disallow: /auth/		 	# login pages
Disallow: *ajax*		 	# ajax pages
Disallow: /channel.html		# my villa
Disallow: /fb/				# fasebook app

User-agent: 008
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: moget
User-agent: ichiro
Disallow: /

User-agent: NaverBot
User-agent: Yeti
Disallow: /

User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: YoudaoBot
Disallow: /

User-agent: Acoon
Disallow: /

User-agent: wotbox
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: psbot
Disallow: /

User-agent: spbot
Disallow: /

Sitemap: https://www.wheretostay.com/sitemap.xml.gz