User-agent: Amazonbot               # Amazon's user agent
Disallow: /					            # disallow this directory

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: *
Crawl-delay: 10

Allow: /$
Allow: /en$
Allow: /fr$
Allow: /en/$
Allow: /fr/$
Allow: /robots.txt

Disallow: */admin/
Disallow: */wizdeo/
Disallow: */widget/

Disallow: */search*?*

Allow: */channels?page=*
Disallow: *channels?page=*&*
Disallow: */channels*?*

Allow: */videos/show/*?page=*
Disallow: */videos/show/*?page=*&*
Disallow: */videos/show/*/

Allow: */videos/view
Disallow: */videos/view/*?*
Disallow: */videos/statistics/*

Disallow: */twitch/*?*


Allow: */channel_lists/view/*?page=*
Disallow: */channel_lists/view/*?page=*&*
Disallow: */channel_lists/view/*/

Disallow */company_claims/

Allow: *.css
Allow: *.js

Disallow: *.json