{ "name": "The Forge census", "version": "2026-11-11", "statement": "The files are the record; site counters are a live reading.", "day_boundary": "UTC. A row belongs to the UTC day its check completed (for a House night: the time the last shard finished).", "generated_from": "Stored data only. Days before a metric's first measured date are absent, never zero-filled. Gross numbers only.", "licence": "Evaluation-only licence: free to use for evaluation, research and reporting, with attribution and a link to theagentforge.io/arena. Not for training or fine-tuning AI models. Model outputs in the run logs remain subject to their providers' terms.", "licence_id": "evaluation-only", "gaps": [ { "from": "2026-11-01", "to": "2026-11-04", "metrics": [ "skill_files.new", "skill_files.scanned_total" ], "status": "not measured", "note": "Lean mode stopped storing skill scans from 1 to 4 Nov 2026, so skill files first scanned on these days were not recorded; totals after 4 Nov leave out files scanned only then." } ], "files": { "series.csv": { "format": "CSV, long format, one row per metric per UTC day", "fields": { "metric": "Metric id (see metrics)", "date_utc": "YYYY-MM-DD, UTC", "value": "Number; empty (null in series.json) when not measured that day (see gaps), never 0", "unit": "What the number counts", "method": "How this row was measured, or \"not measured: \"", "kind": "daily (that day only) or cumulative (everything up to the end of that day)" } }, "series.json": { "format": "JSON: { version, rows: [same fields as series.csv] }" }, "server-history/.csv": { "format": "CSV, one row per server per night per family, one file per UTC month (a complete month's file changes only if its stored rows do)", "fields": { "family": "top500_nightly or weekly_sweep (separate denominators)", "server": "npm: or pypi:", "version": "The version tested (empty if it couldn't be resolved)", "night_utc": "YYYY-MM-DD, the UTC day the run completed", "outcome": "The House tester's raw outcome (worked, partial, failed, no_answer, needs_config, not_on_npm, not_on_pypi, no_command, error)", "failure_class": "The outcome's class in failure-classes/v1; empty when it installed", "tool_count": "Tools listed (empty when it listed none)", "description_sha256": "sha256 (hex) of the tool-description set seen for that server and version on that night; empty when it didn't list tools or before descriptions were captured" } }, "tool-history/.csv": { "format": "CSV, one row per tool-definition set seen per server version (tools/list), one file per UTC month of the version's publish date (or of when we saw it, when the publish date isn't known). Hashes only: the tool text is not in the public files.", "fields": { "server": "The tool's id on the site (npm:, pypi: or the repository it resolves to)", "package": "npm: or pypi: for servers in the backfill set; empty otherwise", "version": "The published version", "published_at": "When the registry says the version was published (npm time, PyPI earliest upload); empty when not known", "observed_at": "When the House tester actually ran that version and read its tools/list", "origin": "nightly (seen by the nightly or weekly House run), carried (the one current row each server had on 5 Nov 2026, copied in), or backfill (an older version installed later, from 2025-09-01). A backfill row's published_at is the registry's date, never a date we saw it", "tools_count": "Tools listed", "description_sha256": "sha256 (hex) of the set's names, descriptions and input schemas as scanned", "change_class": "Against the server's previous observed set: first, none, or tool_added; tool_removed; description_changed; schema_changed (joined by ;). not_yet_computed until the daily drift step has read it", "failure_class": "For a backfill version that didn't start: its class in failure-classes/v1 (the other fields are empty)" }, "coverage": "Backfill: stable versions published since 2025-09-01 of the latest nightly top-500 servers (npm) and up to 100 PyPI servers the weekly sweep has seen start, at most the newest 150 per server. Not tested: prerelease (A prerelease (npx and uvx don't install it by default).); over_cap (Older than the newest 150 stable versions of its server.); no_tool_page (The site has no tool page for this server.)" }, "drift.csv": { "format": "CSV, one row per consecutive pair of observed tool-definition sets of a server whose definitions changed", "fields": { "server": "As in tool-history", "package": "As in tool-history", "from_version": "The earlier version", "to_version": "The later version (equal to from_version when the definitions changed without a version change)", "from_published_at": "Publish date of from_version, where known", "to_published_at": "Publish date of to_version, where known", "change_class": "tool_added; tool_removed; description_changed; schema_changed (joined by ;)", "tools_added": "Tools in the later set only", "tools_removed": "Tools in the earlier set only", "descriptions_changed": "Tools in both whose description text differs", "schemas_changed": "Tools in both whose input schema differs", "major_bump": "true when the two versions' major numbers differ (semver; for PyPI the first release number), in either direction: pairs follow publish order, so a backport to an older major published after a newer one counts as a major change", "zero_major": "true when both versions are 0.x, where semver allows any change in a minor release", "same_version": "true when the definitions changed with no version change", "untested_between": "Versions published between the two that didn't start, so their definitions are unknown", "new_text_rules": "Skill-scanner rule ids (joined by ;) matched by the new text of added or changed tools and not by those tools' earlier text. A rule id names a pattern for a person to read, not a finding about the server" } }, "roster/.csv": { "format": "CSV, the night's top-500 roster", "fields": { "rank": "Rank by last week's downloads", "server": "npm: or pypi:", "version": "Version tested", "weekly_downloads": "Downloads in the week before, as the registry reported them", "outcome": "The night's raw outcome" } }, "manifest.json": { "format": "JSON: the sha256 and size of every file in this version, and every published version with the address of its own manifest" }, "manifest/.json": { "format": "JSON: the sha256 and size of every file in that published version" } }, "families": { "top500_nightly": "The nightly top-500 run. Its denominator is the servers attempted that night; never added to the weekly sweep's.", "weekly_sweep": "The weekly sweep of every no-setup server, a seventh a day. Its denominator is the servers tested that day; never added to the nightly run's.", "tool_descriptions": "Tool descriptions (tools/list) the House tester sent from working servers in either run, as scanned.", "skill_files": "Every distinct skill file the scanner has read (bulk scans and scans by people and agents).", "tools_tracked": "Tools with a page on the site." }, "metrics": [ { "metric": "top500_nightly.servers_attempted", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers tested, counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers tested, counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "top500_nightly.servers_installed", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers that installed, started and answered the MCP initialize handshake (outcome worked or partial), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers that installed, started and answered the MCP initialize handshake (outcome worked or partial), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "top500_nightly.servers_failed", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers tested that did not install and answer (servers_attempted minus installed; includes test_error, our side), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers tested that did not install and answer (servers_attempted minus installed; includes test_error, our side), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "top500_nightly.failed.install_failed", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers that failed with class install_failed (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers that failed with class install_failed (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "top500_nightly.failed.no_handshake", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers that failed with class no_handshake (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers that failed with class no_handshake (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "top500_nightly.failed.needs_config", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers that failed with class needs_config (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers that failed with class needs_config (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "top500_nightly.failed.not_on_registry", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers that failed with class not_on_registry (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers that failed with class not_on_registry (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "top500_nightly.failed.no_command", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers that failed with class no_command (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers that failed with class no_command (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "top500_nightly.failed.test_error", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers that failed with class test_error (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers that failed with class test_error (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "top500_nightly.failed.other", "family": "top500_nightly", "unit": "servers", "kind": "daily", "method": "Servers that failed with class other (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "derived": "Servers that failed with class other (failure-classes/v1), counted from the stored per-server results of the nightly top-500 run (the most-downloaded no-setup servers in the official MCP Registry) (house_results, family top500_nightly) for that UTC night.", "first_measured": "2026-10-15" }, { "metric": "weekly_sweep.servers_tested", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers tested, counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers tested, counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "weekly_sweep.servers_installed", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers that installed, started and answered the MCP initialize handshake (outcome worked or partial), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers that installed, started and answered the MCP initialize handshake (outcome worked or partial), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "weekly_sweep.servers_failed", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers tested that did not install and answer (servers_tested minus installed; includes test_error, our side), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers tested that did not install and answer (servers_tested minus installed; includes test_error, our side), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "weekly_sweep.failed.install_failed", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers that failed with class install_failed (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers that failed with class install_failed (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "weekly_sweep.failed.no_handshake", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers that failed with class no_handshake (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers that failed with class no_handshake (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "weekly_sweep.failed.needs_config", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers that failed with class needs_config (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers that failed with class needs_config (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "weekly_sweep.failed.not_on_registry", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers that failed with class not_on_registry (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers that failed with class not_on_registry (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "weekly_sweep.failed.no_command", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers that failed with class no_command (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers that failed with class no_command (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "weekly_sweep.failed.test_error", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers that failed with class test_error (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers that failed with class test_error (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "weekly_sweep.failed.other", "family": "weekly_sweep", "unit": "servers", "kind": "daily", "method": "Servers that failed with class other (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "derived": "Servers that failed with class other (failure-classes/v1), counted from the stored per-server results of the weekly sweep (a seventh of every no-setup npm and PyPI server each day) (house_results, family weekly_sweep) for that UTC night.", "first_measured": "2026-10-30" }, { "metric": "tool_descriptions.scanned", "family": "tool_descriptions", "unit": "tool descriptions", "kind": "daily", "method": "Non-empty tool descriptions in the description sets the House tester sent that UTC day (sets last seen that day), scanned with the skill scanner's rules. Read by the snapshot just after the day ends.", "derived": null, "first_measured": "2026-11-05" }, { "metric": "tool_descriptions.high_risk_matches", "family": "tool_descriptions", "unit": "servers", "kind": "daily", "method": "Servers whose description set sent that UTC day matched at least one high-severity scanner pattern. A match is a pattern for a person to read, not a finding about the server.", "derived": null, "first_measured": "2026-11-05" }, { "metric": "skill_files.scanned_total", "family": "skill_files", "unit": "files", "kind": "cumulative", "method": "Distinct skill files (sha256 fingerprints in skill_scans) first scanned before the end of that UTC day.", "derived": "Distinct skill files (sha256 fingerprints in skill_scans) first scanned before the end of that UTC day, counted from each file's stored first_seen; listed only on days when at least one file was first scanned.", "first_measured": "2026-10-19" }, { "metric": "skill_files.new", "family": "skill_files", "unit": "files", "kind": "daily", "method": "Distinct skill files first scanned during that UTC day (skill_scans.first_seen).", "derived": "Distinct skill files first scanned during that UTC day, from each file's stored first_seen; listed only on days when at least one file was first scanned.", "first_measured": "2026-10-19" }, { "metric": "tools_tracked.total", "family": "tools_tracked", "unit": "tools", "kind": "cumulative", "method": "Tools with a page (MCP servers, skills and packages, most of them registry listings), created before the end of that UTC day.", "derived": null, "first_measured": "2026-11-05" }, { "metric": "tools_tracked.new", "family": "tools_tracked", "unit": "tools", "kind": "daily", "method": "Tools whose page was created during that UTC day.", "derived": null, "first_measured": "2026-11-05" } ], "failure_classes": { "version": "failure-classes/v1", "closed": true, "classes": { "install_failed": "The package couldn't be installed (not found at that version, or the build or download failed).", "no_handshake": "It installed but never answered the MCP initialize handshake within the time limit.", "needs_config": "It asked for configuration (an API key, token or required setting) that a no-credentials test can't give.", "not_on_registry": "The listed package or version wasn't on npm or PyPI.", "no_command": "The PyPI package has no command of the name the registry gave.", "test_error": "The test itself failed (our side), so nothing is known about the server.", "other": "Any outcome not mapped above (none in v1; kept so the list stays closed)." }, "outcome_map": { "worked": null, "partial": null, "failed": "install_failed", "no_answer": "no_handshake", "needs_config": "needs_config", "not_on_npm": "not_on_registry", "not_on_pypi": "not_on_registry", "no_command": "no_command", "error": "test_error" }, "installed_outcomes": [ "worked", "partial" ] } }