diff options
| author | Christian Cleberg <[email protected]> | 2026-05-01 10:12:52 -0500 |
|---|---|---|
| committer | Christian Cleberg <[email protected]> | 2026-05-01 10:12:52 -0500 |
| commit | 8498733c5778293cc8a1756a0fa4ed3244eb32da (patch) | |
| tree | e27aac98734fd1f5d64922efdd03bf832452cb46 /theme | |
| parent | f36b4ee2df8aa799ceeb033a49c35f41b44fb61d (diff) | |
| download | cleberg.net-8498733c5778293cc8a1756a0fa4ed3244eb32da.tar.gz cleberg.net-8498733c5778293cc8a1756a0fa4ed3244eb32da.tar.bz2 cleberg.net-8498733c5778293cc8a1756a0fa4ed3244eb32da.zip | |
fix: reset robots.txt
Diffstat (limited to 'theme')
| -rw-r--r-- | theme/static/robots.txt | 88 |
1 files changed, 3 insertions, 85 deletions
diff --git a/theme/static/robots.txt b/theme/static/robots.txt index 65785f0..4bd4c71 100644 --- a/theme/static/robots.txt +++ b/theme/static/robots.txt @@ -1,85 +1,3 @@ -# Dark Visitors Robots.txt - -# AI Data Scraper -# https://darkvisitors.com/agents/applebot-extended - -User-agent: Applebot-Extended -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/bytespider - -User-agent: Bytespider -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/ccbot - -User-agent: CCBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/claudebot - -User-agent: ClaudeBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/diffbot - -User-agent: Diffbot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/facebookbot - -User-agent: FacebookBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/google-extended - -User-agent: Google-Extended -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/gptbot - -User-agent: GPTBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/meta-externalagent - -User-agent: Meta-ExternalAgent -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/omgili - -User-agent: omgili -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/timpibot - -User-agent: Timpibot -Disallow: / - -# Undocumented AI Agent -# https://darkvisitors.com/agents/anthropic-ai - -User-agent: anthropic-ai -Disallow: / - -# Undocumented AI Agent -# https://darkvisitors.com/agents/claude-web - -User-agent: Claude-Web -Disallow: / - -# Undocumented AI Agent -# https://darkvisitors.com/agents/cohere-ai - -User-agent: cohere-ai -Disallow: / +User-agent: * +Disallow: +Sitemap: https://cleberg.net/sitemap.xml |