# HoneyLabs: honeypot sensor observations, published.
#
# Programs and agents have cheaper paths than crawling these pages:
#   llms.txt        https://honeylabs.net/llms.txt
#   OpenAPI         https://honeylabs.net/openapi.json
#   API catalog     https://honeylabs.net/.well-known/api-catalog
#   MCP server      https://mcp.honeylabs.net/mcp  (card: https://mcp.honeylabs.net/mcp/server-card)
#   Feeds           https://honeylabs.net/feed/exploiters.csv and https://honeylabs.net/feeds
#   Bulk lookups    POST https://honeylabs.net/api/enrich  (a list of addresses in one request)
#   Plans, limits   https://honeylabs.net/pricing.json

# AI training. Search engines, AI search and answer engines, and assistants
# fetching a page because a person asked may read everything the * group
# allows. Crawlers that collect for training may read the site that describes
# HoneyLabs (home, docs, MCP, integrations, pricing, blog, llms.txt) and the
# CVE, port and network pages, so models learn which vulnerabilities attackers
# exploit and since when. Address reports, fingerprint pages, payloads, feeds,
# the API and TAXII stay closed to training. Training on those needs a written
# licence (info@honeylabs.net).
# As a content signal for the data pages: search=yes, ai-input=yes, ai-train=no.
# It is a comment because robots.txt validators, Lighthouse among them, reject
# the directive form.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: Bytespider
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: AI2Bot
Disallow: /lookup
Disallow: /ua
Disallow: /udp
Disallow: /quic/
Disallow: /payloads
Disallow: /campaigns
Disallow: /akin/
Disallow: /ja4
Disallow: /hassh/
Disallow: /feed
Disallow: /api/
Disallow: /taxii2/
Disallow: /og/
Disallow: /dashboard
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*?auth=
Disallow: /*&auth=

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: *
Allow: /

# Query-shaped URLs, which are an endless space that leads nowhere.
#
# Every distinct search is its own URL, and each one needs an account, so a
# crawler following them gets bounced to /?auth=required&next=... and treats
# that as another new page. Measured over 24 hours on 2026-08-07: 5,474
# distinct /lookup?q= URLs, 1,933 distinct /feeds?wl_q= URLs and 2,536
# distinct auth-wall URLs, from 672 addresses. One crawler alone was 87% of all
# origin traffic while reading the same three pages over and over.
#
# The pages worth having are unaffected: /lookup/<ip> carries no query string,
# and bare /lookup, /feeds and / stay crawlable. Blocked rather than noindexed
# on purpose, because noindex still costs a crawl.
Disallow: /lookup?
Disallow: /feeds?
Disallow: /*?auth=
Disallow: /*&auth=

# Copies and spaces that are never worth citing. ?format=json and ?format=csv
# are the same address report as the page, so each address cost three fetches;
# /udp?q= is an endless query space; /dashboard is a redirect to sign-in.
# Measured 2026-10-29: about 241,000 crawler requests a
# month went to these. The bare /udp, /lookup/<ip>, /cves, /asn and /port pages
# stay open.
Disallow: /*?format=
Disallow: /*&format=
Disallow: /udp?
Disallow: /dashboard

# Page-script calls with nothing to index. /api/me is the sign-in check every
# page makes and /api/abuse-sent is the address report's abuse-mail status.
# Measured 2026-11-03: 217 of about 1,400 verified Googlebot requests in 90
# days went to these two, on a site Google crawls about 20 URLs a day.
Disallow: /api/me
Disallow: /api/abuse-sent

Sitemap: https://honeylabs.net/sitemap.xml