Fastly

Overview

This documentation describes a Fastly Compute@Edge implementation that detects traffic from AI search engines and agentic web crawlers using User-Agent analysis. The function routes AI systems to machine-optimized endpoints, improving how your content is consumed, interpreted, and referenced across the agentic web.

Architecture

Incoming Request → User-Agent Analysis → AI Bot Detection → Routing Decision → URL Rewrite

Key Components

• AI Bot Detection Engine

• Routing Logic

• Configuration Management

• Logging & Observability

AI Search Engine Bot Detection

Introduction

This implementation identifies traffic from AI search engines and agentic web crawlers such as ChatGPT, Perplexity, Gemini, Claude, Grok, and DeepSeek. The system routes these agents to specialized endpoints optimized for AI consumption, improving machine readability, model context accuracy, and your overall presence across the agentic web.

Configuration

AI Bot Pattern Configuration (config.json)

{
  "aiSearchBots": {
    "chatgpt": [
      "chatgpt-user",
      "gptbot",
      "openai-searchbot",
      "chatgpt-search",
      "openai-imagesbot"
    ],
    "gemini": [
      "gemini-bot",
      "google-extended",
      "geminibot",
      "bard",
      "googlebot-ai"
    ],
    "aiOverviews": [
      "google-inspectiontool",
      "google-ai-overview",
      "google-sge"
    ],
    "perplexity": [
      "perplexitybot",
      "perplexity-search",
      "perplexityai"
    ],
    "claude": [
      "claude-web",
      "claudebot",
      "claude-search",
      "anthropic-ai"
    ],
    "grok": [
      "grok-bot",
      "grokbot",
      "xai-bot",
      "x-ai-bot"
    ],
    "deepseek": [
      "deepseekbot",
      "deepseek-search",
      "deepseek-ai"
    ],
    "aiMode": [
      "ai-mode-bot",
      "aimode",
      "intelligent-agent"
    ]
  }
}

Routing Rules Configuration (routing.json)

{
  "routingRules": {
    "chatgpt": {
      "method": "path_prefix",
      "target": "/chatgpt",
      "preserveQuery": true
    },
    "gemini": {
      "method": "path_prefix",
      "target": "/gemini",
      "preserveQuery": true
    },
    "aiOverviews": {
      "method": "path_prefix",
      "target": "/ai-overview",
      "preserveQuery": true
    },
    "perplexity": {
      "method": "path_prefix",
      "target": "/perplexity",
      "preserveQuery": true
    },
    "claude": {
      "method": "path_prefix",
      "target": "/claude",
      "preserveQuery": true
    },
    "grok": {
      "method": "path_prefix",
      "target": "/grok",
      "preserveQuery": true
    },
    "deepseek": {
      "method": "path_prefix",
      "target": "/deepseek",
      "preserveQuery": true
    },
    "aiMode": {
      "method": "path_prefix",
      "target": "/ai-mode",
      "preserveQuery": true
    }
  }
}

Implementation

aiDetection.ts

export type AIBotType =
  | "chatgpt"
  | "gemini"
  | "aiOverviews"
  | "perplexity"
  | "claude"
  | "grok"
  | "deepseek"
  | "aiMode"
  | null;

export function detectAISearchBot(userAgent: string): AIBotType {
  const ua = userAgent.toLowerCase();

  if (ua.includes("chatgpt") || ua.includes("gptbot") || ua.includes("openai"))
    return "chatgpt";

  if (ua.includes("gemini") || ua.includes("bard") || ua.includes("google-extended"))
    return "gemini";

  if (ua.includes("google-sge") || ua.includes("ai-overview"))
    return "aiOverviews";

  if (ua.includes("perplexity"))
    return "perplexity";

  if (ua.includes("claude") || ua.includes("anthropic"))
    return "claude";

  if (ua.includes("grok") || ua.includes("xai-bot"))
    return "grok";

  if (ua.includes("deepseek"))
    return "deepseek";

  if (ua.includes("ai-mode") || ua.includes("intelligent-agent"))
    return "aiMode";

  return null;
}

routing.ts

import type { AIBotType } from "./aiDetection";

interface RoutingRule {
  method: "path_prefix" | "subdomain" | "parameter";
  target: string;
  preserveQuery: boolean;
  param?: string;
  value?: string;
}

interface RoutingConfig {
  routingRules: Record<string, RoutingRule>;
}

export function generateAIBotUrl(
  originalUrl: string,
  aiBot: AIBotType,
  config: RoutingConfig
): string {
  if (!aiBot) return originalUrl;

  const url = new URL(originalUrl);
  const rule = config.routingRules[aiBot];

  if (!rule) return originalUrl;

  switch (rule.method) {
    case "path_prefix":
      url.pathname = rule.target + url.pathname;
      break;
    case "subdomain":
      url.hostname = rule.target;
      break;
    case "parameter":
      url.searchParams.set(rule.param || "ai", rule.value || aiBot);
      break;
  }

  if (!rule.preserveQuery) {
    url.search = "";
  }

  return url.toString();
}

main.ts (Fastly Compute@Edge)

import { detectAISearchBot } from "./aiDetection";
import { generateAIBotUrl } from "./routing";
import routingRules from "./routing.json";

addEventListener("fetch", (event) => event.respondWith(handleRequest(event)));

async function handleRequest(event: FetchEvent): Promise<Response> {
  const request = event.request;
  const userAgent = request.headers.get("user-agent") || "";
  const originalUrl = request.url;

  const aiBot = detectAISearchBot(userAgent);
  if (!aiBot) {
    return fetch(request);
  }

  const targetUrl = generateAIBotUrl(originalUrl, aiBot, { routingRules });

  const newRequest = new Request(targetUrl, request);
  newRequest.headers.set("X-AI-Bot", aiBot);
  newRequest.headers.set("X-Original-URL", originalUrl);
  newRequest.headers.set("X-AI-Optimized", "true");

  return fetch(newRequest);
}

Routing Methods

Path Prefix Routing

/docs/intro → /chatgpt/docs/intro

Subdomain Routing

example.com/page → claude.example.com/page

Parameter Routing

/guide → /guide?ai=perplexity

Monitoring & Analytics

Headers

• X-AI-Bot

• X-Original-URL

• X-AI-Optimized

Log Format

{
  "timestamp": "2025-01-01T12:00:00Z",
  "aiService": "chatgpt",
  "originalUrl": "/article/example",
  "routedUrl": "/chatgpt/article/example",
  "userAgent": "ChatGPT-User/1.0",
  "routingMethod": "path_prefix"
}

Deployment

1. Add files to your Compute@Edge project

2. Build the package

3. Upload & activate your service

4. Verify routing in staging

5. Deploy to production

That’s it! You have now successfully configured Limy router for your website. Data should begin to populate on your dashboard within an hour.