# Allow AdSense to crawl every page (does not affect any other user agent)
User-agent: Mediapartners-Google
Disallow:

# Allow Amazon the same access as Google
User-agent: AmazonAdBot
Disallow:

# Apple very aggressive
User-agent: Applebot
Disallow: /buy.php
Crawl-delay: 5

# Meta very aggressive
User-agent: meta-webindexer
Disallow: /buy.php
Crawl-delay: 5

# PetalBot very aggressive
User-agent: PetalBot
Crawl-delay: 5

# TerraCotta very aggressive
User-agent: TerraCotta
Disallow: /buy.php
Disallow: /albumart/
Crawl-delay: 10

# Block Alibaba
User-agent: AliyunSecBot
Disallow: /

# Block ByteDance (TikTok/Douyin)
User-agent: Bytespider
Disallow: /

# Block Baidu
User-agent: Baiduspider
User-agent: Baiduspider-render
Disallow: /

# Block Alibaba/UC Browser
User-agent: Yisouspider
Disallow: /

# Block Sogou
User-agent: Sogou web spider
User-agent: Sogou inst spider
Disallow: /


# Block Youdao
User-agent: YoudaoBot
Disallow: /

# Block Facebook (this is not the Facebook preview bot)
User-agent: meta-externalagent
Disallow: /

# Universal rules and crawl delay
User-agent: *
Disallow: /cgi-bin/
Disallow: /admin_album.php
Disallow: /admin_band.php
Disallow: /admin_chart.php
Disallow: /latest_mods.php
Disallow: /moderator.php
Disallow: /contact.php
Disallow: /siteupgrade.php
Disallow: /siteerror.php
Disallow: /internal_error.html
Disallow: /forums/login.php
Disallow: /forums/memberlist.php
Disallow: /forums/profile.php

# Disallow crawling some pages with specific parameters
Disallow: /*?*orderby=
Disallow: /*?*sortdir=
Disallow: /*&orderby=
Disallow: /*&sortdir=
Crawl-Delay: 5

# Site maps
Sitemap: https://www.besteveralbums.com/BestEverAlbumsdotCom_SiteMap_Index.xml