# on enleve les répertoire a ne pas crawler
User-agent: *
Disallow: /dynamic/
Disallow: /crm/
Disallow: /cal/
Disallow: /text/
Disallow: /texte/
Disallow: /texto_es/
Disallow: /texto_pt/
Disallow: /owl/
Disallow: /phpmyadmin/
Disallow: /resource/
user-agent: ia_archiver
Disallow: /
user-agent: zeus
Disallow: /
user-agent: crescent
Disallow: /
user-agent: emailsiphon
Disallow: /
user-agent: extractorpro
Disallow: /
user-agent: emailcollector
Disallow: /
user-agent: emailwolf
Disallow: /
user-agent: nicerspro
Disallow: /