87 lines
1.6 KiB
Plaintext
87 lines
1.6 KiB
Plaintext
# robots.txt — mamadaevv.site
|
|
# Портфолио Николая Мамадаева, продуктового дизайнера.
|
|
|
|
Sitemap: https://mamadaevv.site/sitemap.xml
|
|
|
|
# --- Классические поисковые системы ---
|
|
User-agent: *
|
|
Allow: /
|
|
Disallow: /_lab/
|
|
Disallow: /assets/favicon/
|
|
Allow: /assets/
|
|
|
|
# --- Поисковые системы с ИИ-функциями ---
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
User-agent: Applebot-Extended
|
|
Allow: /
|
|
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
|
|
User-agent: DuckAssist
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
# --- Краулеры больших языковых моделей (обучение и ответы) ---
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: OAI-SearchBot
|
|
Allow: /
|
|
|
|
User-agent: ChatGPT-User
|
|
Allow: /
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
|
|
User-agent: Claude-User
|
|
Allow: /
|
|
|
|
User-agent: Google-CloudVertexBot
|
|
Allow: /
|
|
|
|
User-agent: Meta-ExternalAgent
|
|
Allow: /
|
|
|
|
User-agent: Meta-ExternalFetcher
|
|
Allow: /
|
|
|
|
User-agent: MistralAI-User
|
|
Allow: /
|
|
|
|
User-agent: Amazonbot
|
|
Allow: /
|
|
|
|
User-agent: cohere-ai
|
|
Allow: /
|
|
|
|
User-agent: Diffbot
|
|
Allow: /
|
|
|
|
User-agent: Timpibot
|
|
Allow: /
|
|
|
|
# --- Архивирование и исследования ---
|
|
User-agent: CCBot
|
|
Allow: /
|
|
|
|
User-agent: Applebot
|
|
Allow: /
|
|
|
|
# --- Служебные страницы: не индексировать никому ---
|
|
User-agent: *
|
|
Disallow: /cases/template-elements-demo/
|
|
Disallow: /cases/design-system/
|
|
Disallow: /cases/mini-projects-showcase/
|
|
Disallow: /en/cases/template-elements-demo/
|
|
Disallow: /en/cases/design-system/
|
|
Disallow: /en/cases/mini-projects-showcase/
|