User-agent: *

#Dynamic URLs

Disallow: /*?s=
Disallow: /*?industry=
Disallow: /*?region=
Disallow: /*?business=
Disallow: /*?solution=
Disallow: /*sortby=
Disallow: /*?action=
Disallow: /*?year=

#Paths & URls

Disallow: /privacy/embed/
Disallow: */media_category
Disallow: */email-preference-center/
Disallow: */process/*

# Prevents issues with GWT

Allow: /*.css$
Allow: /*.js$

#Trackbacks

Disallow: /trackback
Disallow: /*trackback
Disallow: /*trackback*
Disallow: /*/trackback

#Block feeds for crawlers

Allow: /feed/$
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /*/feed/$
Disallow: /*/feed/rss/$
Disallow: /*/trackback/$
Disallow: /*/*/feed/$
Disallow: /*/*/feed/rss/$
Disallow: /*/*/trackback/$
Disallow: /*/*/*/feed/$
Disallow: /*/*/*/feed/rss/$
Disallow: /*/*/*/trackback/$

#Bots list to exclude

User-agent: MSIECrawler
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: libwww
Disallow: /

Sitemap: https://www.criteo.com/sitemap_index.xml
Sitemap: https://www.criteo.com/de/sitemap_index.xml
Sitemap: https://www.criteo.com/fr/sitemap_index.xml
Sitemap: https://www.criteo.com/br/sitemap_index.xml
Sitemap: https://www.criteo.com/jp/sitemap_index.xml
Sitemap: https://www.criteo.com/kr/sitemap_index.xml
Sitemap: https://www.criteo.com/es/sitemap_index.xml
Sitemap: https://www.criteo.com/it/sitemap_index.xml
Sitemap: https://www.criteo.com/ru/sitemap_index.xml