# /robots.txt

User-agent: #
Disallow: /email_addresses/

User-agent: SBIder
Disallow: /

User-agent: lwp-trivial
Disallow: /

User-agent: lwp-trivial/1.35
Disallow: /

User-agent: lwp-trivial/1.38
Disallow: /

User-agent: lwp-trivial/1.4
Disallow: /

User-agent: ichiro
Disallow: /

# Exclude unimportant areas

User-agent: *
Disallow: /assets/

User-agent: *
Disallow: /codebase/