# Global rules for all bots
User-agent: *

# Allow first page of paginated content
Allow: /*?page=1$
Allow: /*&page=1$

# Django Admin and authentication
Disallow: /admin/
Disallow: /accounts/
Disallow: /password-reset/
Disallow: /verify-email/
Disallow: /auth/callback/
Disallow: /social-login/

# MFA related paths
Disallow: /verify-mfa/
Disallow: /setup-mfa/
Disallow: /disable-mfa/

# Search and dynamic content
Disallow: /*?query=
Disallow: /companies/search/
Disallow: /companies/get_choices/
Disallow: /company-autocomplete/

# API endpoints
Disallow: /api/
Disallow: /*/api/

# --- Filings Rules ---
# Allow crawling of processed HTML content pages
Allow: /filings/*/content/$
# Disallow utility/action endpoints under filings
Disallow: /filings/process-filings-austria/$
Disallow: /filings/ask-gemini/$
# Optional: Disallow list views if they are not valuable for SEO
# Disallow: /filings/$
# Disallow: /filings/filing-types/$

# --- Media and Static Files ---
Allow: /media/companies/      # Allow company logos and images
Allow: /media/blog/          # Allow blog images
# Block raw filing files in media directories
Disallow: /media/filings/
Disallow: /financialreports/media/filings/
# Block static files
Disallow: /static/
Disallow: /financialreports/static/

# Application-specific paths (excluding those now handled in Filings Rules)
Disallow: /insider/*/edit/
Disallow: /publications/preview/
Disallow: /helpcenter/draft/
Disallow: /esef/validate/
Disallow: /shortsells/update/

# Protected and utility paths
Disallow: /protected-file/
Disallow: /companies/protected-file/
Disallow: /*/protected-file/
Disallow: /process-filings-austria/ # Keep as safety, might be redundant
Disallow: /*get_filtered_filings/
Disallow: /*load_more_trades/

# User-related paths
Disallow: /users/settings/
Disallow: /users/profile/edit/
Disallow: /users/password/

# Pagination (disallow all pages except first page)
Disallow: /*?page=
Disallow: /*&page=

# Sitemaps
Sitemap: https://financialreports.eu/sitemap.xml

# --- AI Bots ---
# No specific rules here means GPTBot, CCBot, ChatGPT-User, etc.,
# will follow the User-agent: * rules above.

# Google Ads Bot
User-agent: AdsBot-Google
Allow: /

# Specific rules for major search engines (Refined)
# Only explicitly repeating rules needed if different from *
# Keeping media rules explicit for clarity/safety.
User-agent: Googlebot
Allow: /media/companies/
Allow: /media/blog/
Disallow: /media/filings/
Disallow: /financialreports/media/filings/

User-agent: bingbot
Allow: /media/companies/
Allow: /media/blog/
Disallow: /media/filings/
Disallow: /financialreports/media/filings/