How to Block AI Bots on Elixir Plug: Complete 2026 Guide

Step 1 — Shared bot list (lib/my_app/ai_bots.ex)

Module attribute @bots is a compile-time constant — the list is baked into the BEAM bytecode. Pattern-match ai_bot?(nil) separately so callers don't need to guard against missing headers.

# lib/my_app/ai_bots.ex — shared bot list

defmodule MyApp.AiBots do
  @moduledoc "AI bot user-agent patterns for blocking."

  @bots [
    # OpenAI
    "gptbot", "chatgpt-user", "oai-searchbot",
    # Anthropic
    "claudebot", "claude-web",
    # Common Crawl
    "ccbot",
    # Bytedance
    "bytespider",
    # Meta
    "meta-externalagent",
    # Perplexity
    "perplexitybot",
    # Google AI
    "google-extended", "googleother",
    # Cohere
    "cohere-ai",
    # Amazon
    "amazonbot",
    # Diffbot
    "diffbot",
    # AI2
    "ai2bot",
    # DeepSeek
    "deepseekbot",
    # Mistral
    "mistralai-user",
    # xAI
    "xai-bot",
    # You.com
    "youbot",
    # DuckDuckGo AI
    "duckassistbot",
  ]

  @doc "Returns true if the user-agent string matches a known AI bot."
  def ai_bot?(nil), do: false
  def ai_bot?(ua) when is_binary(ua) do
    ua_lower = String.downcase(ua)
    Enum.any?(@bots, &String.contains?(ua_lower, &1))
  end
end

Step 2 — The bot-blocking Plug module

get_req_header/2 returns a list — headers can appear multiple times. Use List.first(headers, "") to safely default to an empty string. put_resp_header/3 before send_resp/3 ensures the header appears on both blocked (403) and legitimate responses.

# lib/my_app/plugs/ai_bot_blocker.ex — the Plug module

defmodule MyApp.Plugs.AiBotBlocker do
  @moduledoc """
  Plug that blocks AI training bots with a 403.

  Usage in a Plug.Builder pipeline:
    plug MyApp.Plugs.AiBotBlocker

  Usage in a Plug.Router:
    plug MyApp.Plugs.AiBotBlocker
    get "/", do: send_resp(conn, 200, "ok")
  """

  import Plug.Conn
  alias MyApp.AiBots

  # init/1 runs at compile time — parse options here, not in call/2.
  # Returning opts unchanged is the common pattern for simple plugs.
  def init(opts), do: opts

  # call/2 runs on every request.
  # Returning conn passes through to the next plug.
  # Calling halt(conn) after send_resp stops the pipeline — no downstream
  # plugs run, including route handlers.
  def call(conn, _opts) do
    ua =
      conn
      |> get_req_header("user-agent")
      |> List.first("")  # headers return a list; default to "" if absent

    if AiBots.ai_bot?(ua) do
      conn
      |> put_resp_header("x-robots-tag", "noai, noimageai")
      |> send_resp(403, "Forbidden")
      |> halt()   # ← stops the pipeline; no handler ever runs
    else
      # Add X-Robots-Tag to all legitimate responses too
      put_resp_header(conn, "x-robots-tag", "noai, noimageai")
      # Return conn unchanged — next plug in the pipeline runs
    end
  end
end

Step 3 — Global pipeline with Plug.Builder

use Plug.Builder gives you the plug macro. Order is execution order: Plug.Static first (so robots.txt is served without hitting the bot blocker), then logging, then bot blocking, then route dispatch. The :match and :dispatch plugs are only needed if you're using Plug.Router — skip them in plain Builder.

# lib/my_app/router.ex — global pipeline with Plug.Builder

defmodule MyApp.Router do
  use Plug.Builder

  # 1. Static files first — robots.txt never hits the bot blocker.
  #    Plug.Static calls halt() itself when it serves a file.
  plug Plug.Static,
    at: "/",
    from: "priv/static",
    only: ["robots.txt", "favicon.ico"]

  # 2. Request logger (before blocking so all requests are logged)
  plug Plug.Logger

  # 3. AI bot blocker — halts the pipeline for AI bots with 403
  plug MyApp.Plugs.AiBotBlocker

  # 4. Route matching — only runs if not halted
  plug :dispatch

  def dispatch(conn, _opts) do
    case conn.request_path do
      "/" ->
        send_resp(conn, 200, "Welcome")

      "/health" ->
        send_resp(conn, 200, "ok")

      "/api/data" ->
        conn
        |> put_resp_content_type("application/json")
        |> send_resp(200, ~s({"data": "protected"}))

      _ ->
        send_resp(conn, 404, "Not Found")
    end
  end
end

Step 4 — Scoped blocking with Plug.Router and forward/2

forward "/api", to: MyApp.ApiRouter delegates all /api/* requests to a sub-router with its own pipeline — including its own bot blocker. Public routes (/health, /robots.txt) remain unprotected.

# lib/my_app/router.ex — scoped blocking with Plug.Router

defmodule MyApp.Router do
  use Plug.Router

  # Plug.Static for robots.txt — runs before everything
  plug Plug.Static,
    at: "/",
    from: "priv/static",
    only: ["robots.txt"]

  # Global pipeline — runs on all requests
  plug :match
  plug :dispatch

  # Public routes — no bot blocker
  get "/health" do
    send_resp(conn, 200, "ok")
  end

  # Forward /api/* to the protected sub-router
  # The sub-router has its own pipeline with AiBotBlocker
  forward "/api", to: MyApp.ApiRouter

  # Catch-all
  match _ do
    send_resp(conn, 404, "Not Found")
  end
end

# Protected API sub-router
defmodule MyApp.ApiRouter do
  use Plug.Router

  # Bot blocker applied only to /api/* routes
  plug MyApp.Plugs.AiBotBlocker

  plug :match
  plug :dispatch

  get "/data" do
    conn
    |> put_resp_content_type("application/json")
    |> send_resp(200, ~s({"data": "protected"}))
  end

  get "/users" do
    conn
    |> put_resp_content_type("application/json")
    |> send_resp(200, ~s([]))
  end

  match _ do
    send_resp(conn, 404, "Not Found")
  end
end

Step 5 — Application supervisor (Bandit or Cowboy)

Plug is server-agnostic. Bandit is the modern choice — pure Elixir, HTTP/2, WebSocket support, and default in Phoenix 1.7+. Cowboy remains the battle-tested option for high-throughput production deployments. Switching servers requires only a one-line change in application.ex.

# lib/my_app/application.ex — start with Bandit or Cowboy

defmodule MyApp.Application do
  use Application

  def start(_type, _args) do
    children = [
      # Option A: Bandit (pure Elixir, HTTP/1 + HTTP/2 + WebSocket)
      # mix.exs: {:bandit, "~> 1.0"}
      {Bandit, plug: MyApp.Router, port: 4000},

      # Option B: Cowboy (Erlang, battle-tested)
      # mix.exs: {:plug_cowboy, "~> 2.0"}
      # {Plug.Cowboy, scheme: :http, plug: MyApp.Router, port: 4000},
    ]

    opts = [strategy: :one_for_one, name: MyApp.Supervisor]
    Supervisor.start_link(children, opts)
  end
end

# mix.exs dependencies:
# defp deps do
#   [
#     {:plug, "~> 1.16"},
#     {:bandit, "~> 1.0"},       # or {:plug_cowboy, "~> 2.0"}
#     {:jason, "~> 1.4"},        # optional — JSON encoding
#   ]
# end

Step 6 — robots.txt

Place robots.txt in priv/static/. Plug.Static serves it and calls halt() itself — your bot blocker never runs for this path. Always register Plug.Static before the bot blocker in the pipeline.

# priv/static/robots.txt — place this file in your static directory
# Plug.Static serves it at GET /robots.txt automatically.

User-agent: *
Allow: /

# AI training bots — blocked
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: AmazonBot
Disallow: /

User-agent: Diffbot
Disallow: /


# In your pipeline (Plug.Builder or Plug.Router):
# plug Plug.Static, at: "/", from: "priv/static", only: ["robots.txt"]
#
# Plug.Static must come BEFORE AiBotBlocker in the pipeline.
# It calls halt() when it serves a file, so the bot blocker
# never runs for /robots.txt requests — legitimate crawlers
# can always read it.

Step 7 — noai meta tag in EEx templates

# noai meta tag in EEx templates (without Phoenix)
# lib/my_app/templates/layout.html.eex

<!DOCTYPE html>
<html>
<head>
  <meta name="robots" content="noai, noimageai">
  <title><%= @title %></title>
</head>
<body>
  <%= @inner_content %>
</body>
</html>

# Rendering from a Plug handler:
defmodule MyApp.Router do
  use Plug.Router

  plug :match
  plug :dispatch

  get "/" do
    html = EEx.eval_file("lib/my_app/templates/layout.html.eex",
      assigns: [title: "My Site", inner_content: "<h1>Welcome</h1>"]
    )
    conn
    |> put_resp_content_type("text/html")
    |> put_resp_header("x-robots-tag", "noai, noimageai")
    |> send_resp(200, html)
  end
end

Plug vs Phoenix Endpoint vs Plug.Router vs Cowboy