# This says to apply these settings to ALL search engine spiders/crawlers

User-agent: IRLbot
Disallow: /

User-agent: Twiceler
Disallow: /

User-agent: Twiceler-0.9
Disallow: /



User-agent: *

# These settings will keep spiders from indexing your unwanted pages 
Disallow: /private_site
Disallow: /catalog/admin
Disallow: /catalog/account.php
Disallow: /catalog/advanced_search.php
Disallow: /catalog/checkout_shipping.php
Disallow: /catalog/create_account.php
Disallow: /catalog/login.php
Disallow: /catalog/password_forgotten.php
Disallow: /catalog/shopping_cart.php
Disallow: /catalog/reviews.php
Disallow: /catalog/product_reviews.php
Disallow: /catalog/products_new.php
Disallow: /catalog/cookie_usage.php
Disallow: /catalog/contact_us.php 
Disallow: /catalog/account_notifications.php