# Robots.txt per atlantdnps.com
# Consulenza Digitale Milano

# Regole generali per tutti i bot
User-agent: *
Allow: /

# Cartelle e file consentiti
Allow: /img/
Allow: /css/
Allow: /js/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$

# File importanti consentiti
Allow: /manifest.json
Allow: /sitemap.xml
Allow: /privacy-policy.html
Allow: /termini-di-utilizzo.html
Allow: /chi-siamo.html

# Disallow cartelle amministrative e temporanee
Disallow: /admin/
Disallow: /tmp/
Disallow: /temp/
Disallow: /backup/
Disallow: /logs/
Disallow: /.git/
Disallow: /.htaccess
Disallow: /error_log
Disallow: /cgi-bin/

# Disallow file di configurazione
Disallow: /.env
Disallow: /config/
Disallow: /database/
Disallow: /vendor/
Disallow: /node_modules/

# Disallow pagine di test e sviluppo
Disallow: /test/
Disallow: /dev/
Disallow: /staging/
Disallow: /*?test=*
Disallow: /*?debug=*

# Parametri URL da escludere
Disallow: /*?utm_*
Disallow: /*?ref=*
Disallow: /*?affiliate=*
Disallow: /*?campaign=*

# Regole specifiche per Googlebot
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Regole per Googlebot Images
User-agent: Googlebot-Image
Allow: /img/
Allow: /*.jpg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$

# Regole per Bingbot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Regole per altri motori importanti
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: YandexBot
Allow: /
Crawl-delay: 2

# Regole per bot social media
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: Applebot
Allow: /
Allow: /img/logo.png
Allow: /privacy-policy.html
Allow: /chi-siamo.html

# Regole per bot di analisi SEO
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: MajesticSEO
Crawl-delay: 10

# Bot da bloccare (spam e scrapers)
User-agent: ia_archiver
User-agent: Wayback
User-agent: archive.org_bot
User-agent: EmailCollector
User-agent: WebReaper
User-agent: WebCopier
User-agent: Offline Explorer
User-agent: HTTrack
User-agent: Microsoft URL Control
User-agent: EmailSiphon
User-agent: BullsEye
User-agent: InfoNaviRobot
User-agent: EmailWolf
User-agent: ExtractorPro
User-agent: CherryPicker
User-agent: WebBandit
User-agent: NICErsPRO
User-agent: VoidEYE
User-agent: Crescent
User-agent: SiteSnagger
User-agent: ProWebWalker
User-agent: CheeseBot
User-agent: WebZip
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebStripper
User-agent: WebSauger
User-agent: WebCopier
User-agent: NetAnts
User-agent: Mister PiX
User-agent: WebAuto
User-agent: TheNomad
User-agent: WWW-Collector-E
User-agent: RMA
User-agent: libWeb
User-agent: asterias
User-agent: httplib
User-agent: turingos
User-agent: spanner
User-agent: InfoNaviRobot
User-agent: Harvest
User-agent: audit
User-agent: Digincore
User-agent: WebSauger
User-agent: Teleport
User-agent: TeleportPro
User-agent: MIIxpc
User-agent: Telesoft
User-agent: Website Quester
User-agent: WebZIP
User-agent: moget
User-agent: WebReaper
User-agent: WebSauger
User-agent: WebCopier
User-agent: NetAnts
User-agent: Papa Foto
User-agent: WebCapture
User-agent: WebCopier v.2.2
User-agent: Offline Explorer
Disallow: /

# Sitemap
Sitemap: https://atlantdnps.com/sitemap.xml

# Host preferito (canonical)
Host: https://atlantdnps.com

# Note per webmaster
# Questo file robots.txt è stato ottimizzato per:
# - Consentire l'indicizzazione di tutti i contenuti pubblici
# - Bloccare bot spam e scrapers
# - Ottimizzare il crawl budget
# - Supportare tutti i principali motori di ricerca
# - Proteggere file sensibili e directory amministrative
#
# Ultimo aggiornamento: 22 luglio 2025
# Contatto: info@atlantdnps.com