{"schemaVersion":"agentstaffer.trial-queue.v0","generatedAt":"2026-10-17T17:25:14.624Z","canonicalUrl":"https://agentstaffer.com/trials.json?q=evaluating-llms-harness&sort=trial_readiness","htmlUrl":"https://agentstaffer.com/trials?q=evaluating-llms-harness&sort=trial_readiness","crawlerEntrypoints":{"schemaVersion":"agentstaffer.crawler-entrypoints.v1","sourcePath":"/trials.json","purpose":"Cloudflare shows /trials and /trials.json are active follow-on crawler surfaces; use this field to recover GEO orientation before interpreting trial evidence or promotion gates.","cloudflareSignal":{"observedAt":"2026-06-16T17:08:40.021Z","window":{"start":"2026-06-15T17:08:34.785Z","end":"2026-06-16T17:08:34.785Z","label":"June 15, 2026 10:08 PDT to June 16, 2026 10:08 PDT"},"currentTrackedAiSearchRequests":15000,"finding":"Cloudflare shows /market.json, /compare.json, /api/market-lite, and /trials.json remain active, while /llms.txt and /geo.json had zero tracked AI/search requests in the current 24h window."},"requiredOrientation":[{"path":"/llms.txt","url":"https://agentstaffer.com/llms.txt"},{"path":"/geo.json","url":"https://agentstaffer.com/geo.json"},{"path":"/agent-visibility.json","url":"https://agentstaffer.com/agent-visibility.json"},{"path":"/openai.json","url":"https://agentstaffer.com/openai.json"},{"path":"/ai-crawl-monitor.json","url":"https://agentstaffer.com/ai-crawl-monitor.json"}],"recommendedNextFetches":[{"path":"/promotions.json","url":"https://agentstaffer.com/promotions.json"},{"path":"/compare.json","url":"https://agentstaffer.com/compare.json"},{"path":"/market.json","url":"https://agentstaffer.com/market.json"},{"path":"/agents.json","url":"https://agentstaffer.com/agents.json"},{"path":"/resources.json","url":"https://agentstaffer.com/resources.json"},{"path":"/retrieval.json","url":"https://agentstaffer.com/retrieval.json"}],"crawlerSpecificNextFetches":[{"crawler":"Claude-SearchBot / ClaudeBot","fetch":[{"path":"/geo.json","url":"https://agentstaffer.com/geo.json"},{"path":"/compare.json","url":"https://agentstaffer.com/compare.json"},{"path":"/trials.json","url":"https://agentstaffer.com/trials.json"}]},{"crawler":"OAI-SearchBot","fetch":[{"path":"/openai.json","url":"https://agentstaffer.com/openai.json"},{"path":"/answers.json","url":"https://agentstaffer.com/answers.json"},{"path":"/market.json","url":"https://agentstaffer.com/market.json"},{"path":"/compare.json","url":"https://agentstaffer.com/compare.json"},{"path":"/sitemap.xml","url":"https://agentstaffer.com/sitemap.xml"}]},{"crawler":"ChatGPT-User","fetch":[{"path":"/openai.json","url":"https://agentstaffer.com/openai.json"},{"path":"/llms.txt","url":"https://agentstaffer.com/llms.txt"},{"path":"/answers.json","url":"https://agentstaffer.com/answers.json"},{"path":"/compare.json","url":"https://agentstaffer.com/compare.json"},{"path":"/trials.json","url":"https://agentstaffer.com/trials.json"}]},{"crawler":"PerplexityBot","fetch":[{"path":"/perplexity.json","url":"https://agentstaffer.com/perplexity.json"},{"path":"/geo.json","url":"https://agentstaffer.com/geo.json"},{"path":"/retrieval.json","url":"https://agentstaffer.com/retrieval.json"},{"path":"/compare.json","url":"https://agentstaffer.com/compare.json"}]},{"crawler":"Googlebot / GoogleOther","fetch":[{"path":"/sitemap.xml","url":"https://agentstaffer.com/sitemap.xml"},{"path":"/market.json","url":"https://agentstaffer.com/market.json"},{"path":"/resources.json","url":"https://agentstaffer.com/resources.json"}]}],"answerRule":"Use this entrypoint for the immediate task, then fetch the requiredOrientation resources before broad crawling or citing AgentStaffer recommendations."},"count":6,"trialQueue":[{"id":"explainx:skills-davila7-claude-code-templates-evaluating-llms-harness","slug":"explainx-evaluating-llms-harness","name":"evaluating-llms-harness","type":"skill","source":"explainx","status":"pending","summary":"lm-evaluation-harness evaluates LLMs across 60+ academic benchmarks using standardized prompts and metrics.","profileUrl":"https://agentstaffer.com/skills/explainx-evaluating-llms-harness","fitScore":61,"roleFamily":"Coding & Development","reviewSignal":"Pending","urls":{"profileJson":"https://agentstaffer.com/skills/explainx-evaluating-llms-harness/profile.json","installJson":"https://agentstaffer.com/skills/explainx-evaluating-llms-harness/install.json","trustJson":"https://agentstaffer.com/skills/explainx-evaluating-llms-harness/trust.json","trialJson":"https://agentstaffer.com/skills/explainx-evaluating-llms-harness/trial.json"},"queueRank":1,"recommendation":{"label":"Needs review","score":61},"trialPriority":81,"trial":{"title":"Run a small code review trial","input":"Give the agent one low-risk issue or one small pull request and ask for defects, tests, and a patch plan.","expectedOutput":"A concise review with reproducible findings, changed-file scope, and test suggestions.","safetyNotes":["Run in a sandbox or throwaway workspace.","Use non-production data and temporary credentials.","Capture output, logs, install friction, and any permission requests."],"decisionCriteria":["Finds concrete issues","Avoids broad rewrites","Explains tradeoffs","Can be evaluated without secrets"],"runUrl":"https://agentstaffer.com/skills/explainx-evaluating-llms-harness#trial","runPath":"/skills/explainx-evaluating-llms-harness#trial","createEndpoint":"https://agentstaffer.com/api/actions/run-trial","createBody":{"listingId":"explainx:skills-davila7-claude-code-templates-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}},"launchPacket":{"schemaVersion":"agentstaffer.trial-launch-packet-lite.v1","requiredPermissions":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"mcp-client","label":"MCP-capable client","detail":"Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","severity":"required","text":"MCP-capable client: Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs."},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required","text":"Local terminal: A CLI path is visible, but the exact command must be verified from source."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","MCP-capable client: Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","Local terminal: A CLI path is visible, but the exact command must be verified from source.","License review: License metadata is missing from the current public profile."]},"preflightGates":[{"id":"profile","label":"Profile record","href":"https://agentstaffer.com/skills/explainx-evaluating-llms-harness/profile.json","passed":true},{"id":"install","label":"Setup guide","href":"https://agentstaffer.com/skills/explainx-evaluating-llms-harness/install.json","passed":true},{"id":"sandbox","label":"Use sandbox data","passed":true}],"promotionGates":[{"id":"install","label":"Install path reviewed","passed":true},{"id":"trust","label":"Trust evidence reviewed","passed":false},{"id":"trial","label":"Sandbox trial recorded","passed":false}],"stopConditions":["Install asks for production credentials or broad write access.","Source or package cannot be verified from profile evidence."],"createRequest":{"endpoint":"https://agentstaffer.com/api/actions/run-trial","method":"POST","body":{"listingId":"explainx:skills-davila7-claude-code-templates-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}}},"safety":{"trustTier":"Needs review","installReadiness":"MCP ready","topRisk":{"level":"medium","label":"No community evaluation","detail":"AgentStaffer users have not submitted a review yet."},"requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required"},{"id":"mcp-client","label":"MCP-capable client","detail":"Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","severity":"required"},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required"},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review"}],"permissionSummary":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"mcp-client","label":"MCP-capable client","detail":"Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","severity":"required","text":"MCP-capable client: Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs."},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required","text":"Local terminal: A CLI path is visible, but the exact command must be verified from source."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","MCP-capable client: Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","Local terminal: A CLI path is visible, but the exact command must be verified from source.","License review: License metadata is missing from the current public profile."]}}},{"id":"mcpservers:agent-skills-firecrawl-evaluating-llms-harness","slug":"mcpservers-evaluating-llms-harness","name":"evaluating-llms-harness","type":"skill","source":"mcpservers","status":"pending","summary":"Evaluates LLMs across 60+ academic benchmarks (MMLU, HumanEval, GSM8K, TruthfulQA, HellaSwag). Use when benchmarking model quality, comparing models, reporting…","profileUrl":"https://agentstaffer.com/skills/mcpservers-evaluating-llms-harness","fitScore":61,"roleFamily":"Coding & Development","reviewSignal":"Pending","urls":{"profileJson":"https://agentstaffer.com/skills/mcpservers-evaluating-llms-harness/profile.json","installJson":"https://agentstaffer.com/skills/mcpservers-evaluating-llms-harness/install.json","trustJson":"https://agentstaffer.com/skills/mcpservers-evaluating-llms-harness/trust.json","trialJson":"https://agentstaffer.com/skills/mcpservers-evaluating-llms-harness/trial.json"},"queueRank":2,"recommendation":{"label":"Needs review","score":61},"trialPriority":81,"trial":{"title":"Run a small code review trial","input":"Give the agent one low-risk issue or one small pull request and ask for defects, tests, and a patch plan.","expectedOutput":"A concise review with reproducible findings, changed-file scope, and test suggestions.","safetyNotes":["Run in a sandbox or throwaway workspace.","Use non-production data and temporary credentials.","Capture output, logs, install friction, and any permission requests."],"decisionCriteria":["Finds concrete issues","Avoids broad rewrites","Explains tradeoffs","Can be evaluated without secrets"],"runUrl":"https://agentstaffer.com/skills/mcpservers-evaluating-llms-harness#trial","runPath":"/skills/mcpservers-evaluating-llms-harness#trial","createEndpoint":"https://agentstaffer.com/api/actions/run-trial","createBody":{"listingId":"mcpservers:agent-skills-firecrawl-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}},"launchPacket":{"schemaVersion":"agentstaffer.trial-launch-packet-lite.v1","requiredPermissions":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"mcp-client","label":"MCP-capable client","detail":"Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","severity":"required","text":"MCP-capable client: Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs."},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required","text":"Local terminal: A CLI path is visible, but the exact command must be verified from source."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","MCP-capable client: Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","Local terminal: A CLI path is visible, but the exact command must be verified from source.","License review: License metadata is missing from the current public profile."]},"preflightGates":[{"id":"profile","label":"Profile record","href":"https://agentstaffer.com/skills/mcpservers-evaluating-llms-harness/profile.json","passed":true},{"id":"install","label":"Setup guide","href":"https://agentstaffer.com/skills/mcpservers-evaluating-llms-harness/install.json","passed":true},{"id":"sandbox","label":"Use sandbox data","passed":true}],"promotionGates":[{"id":"install","label":"Install path reviewed","passed":true},{"id":"trust","label":"Trust evidence reviewed","passed":false},{"id":"trial","label":"Sandbox trial recorded","passed":false}],"stopConditions":["Install asks for production credentials or broad write access.","Source or package cannot be verified from profile evidence."],"createRequest":{"endpoint":"https://agentstaffer.com/api/actions/run-trial","method":"POST","body":{"listingId":"mcpservers:agent-skills-firecrawl-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}}},"safety":{"trustTier":"Needs review","installReadiness":"MCP ready","topRisk":{"level":"medium","label":"No community evaluation","detail":"AgentStaffer users have not submitted a review yet."},"requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required"},{"id":"mcp-client","label":"MCP-capable client","detail":"Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","severity":"required"},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required"},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review"}],"permissionSummary":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"mcp-client","label":"MCP-capable client","detail":"Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","severity":"required","text":"MCP-capable client: Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs."},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required","text":"Local terminal: A CLI path is visible, but the exact command must be verified from source."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","MCP-capable client: Use Claude Desktop, Cursor, VS Code, or another client that can load MCP server configs.","Local terminal: A CLI path is visible, but the exact command must be verified from source.","License review: License metadata is missing from the current public profile."]}}},{"id":"claudemarketplaces:skills-orchestra-research-ai-research-skills-evaluating-llms-harness","slug":"claudemarketplaces-evaluating-llms-harness","name":"evaluating-llms-harness","type":"skill","source":"claudemarketplaces","status":"pending","summary":"This wraps the LM Evaluation Harness, the standard tool for benchmarking language models across 60+ academic datasets like MMLU, GSM8K, and HellaSwag. You point it at any HuggingFace model and get apples-to-apples comparisons using consistent prompts and metrics. It's what most researchers use when they publish benchmark numbers, so if you're fine-tuning mo...","profileUrl":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness","fitScore":50,"roleFamily":"Coding & Development","reviewSignal":"Pending","urls":{"profileJson":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/profile.json","installJson":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/install.json","trustJson":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/trust.json","trialJson":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/trial.json"},"queueRank":3,"recommendation":{"label":"Needs review","score":50},"trialPriority":70,"trial":{"title":"Run a small code review trial","input":"Give the agent one low-risk issue or one small pull request and ask for defects, tests, and a patch plan.","expectedOutput":"A concise review with reproducible findings, changed-file scope, and test suggestions.","safetyNotes":["Run in a sandbox or throwaway workspace.","Use non-production data and temporary credentials.","Capture output, logs, install friction, and any permission requests."],"decisionCriteria":["Finds concrete issues","Avoids broad rewrites","Explains tradeoffs","Can be evaluated without secrets"],"runUrl":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness#trial","runPath":"/skills/claudemarketplaces-evaluating-llms-harness#trial","createEndpoint":"https://agentstaffer.com/api/actions/run-trial","createBody":{"listingId":"claudemarketplaces:skills-orchestra-research-ai-research-skills-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}},"launchPacket":{"schemaVersion":"agentstaffer.trial-launch-packet-lite.v1","requiredPermissions":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required","text":"Local terminal: A CLI path is visible, but the exact command must be verified from source."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","Local terminal: A CLI path is visible, but the exact command must be verified from source.","License review: License metadata is missing from the current public profile."]},"preflightGates":[{"id":"profile","label":"Profile record","href":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/profile.json","passed":true},{"id":"install","label":"Setup guide","href":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/install.json","passed":true},{"id":"sandbox","label":"Use sandbox data","passed":true}],"promotionGates":[{"id":"install","label":"Install path reviewed","passed":true},{"id":"trust","label":"Trust evidence reviewed","passed":false},{"id":"trial","label":"Sandbox trial recorded","passed":false}],"stopConditions":["Install asks for production credentials or broad write access.","Source or package cannot be verified from profile evidence."],"createRequest":{"endpoint":"https://agentstaffer.com/api/actions/run-trial","method":"POST","body":{"listingId":"claudemarketplaces:skills-orchestra-research-ai-research-skills-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}}},"safety":{"trustTier":"Needs review","installReadiness":"CLI likely","topRisk":{"level":"medium","label":"No community evaluation","detail":"AgentStaffer users have not submitted a review yet."},"requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required"},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required"},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review"}],"permissionSummary":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required","text":"Local terminal: A CLI path is visible, but the exact command must be verified from source."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","Local terminal: A CLI path is visible, but the exact command must be verified from source.","License review: License metadata is missing from the current public profile."]}}},{"id":"remoteopenclaw:skills-hermes-built-in-evaluating-llms-harness","slug":"remoteopenclaw-evaluating-llms-harness","name":"Evaluating Llms Harness","type":"skill","source":"remoteopenclaw","status":"pending","summary":"Evaluates LLMs across 60+ academic benchmarks (MMLU, HumanEval, GSM8K, TruthfulQA, HellaSwag). Use when benchmarking model quality, comparing models, reporting academic results, or tracking training progress. Industry standard used by EleutherAI, HuggingFace, and major labs. Supports HuggingFace, vLLM, APIs.","profileUrl":"https://agentstaffer.com/skills/remoteopenclaw-evaluating-llms-harness","fitScore":38,"roleFamily":"Research & Knowledge","reviewSignal":"Pending","urls":{"profileJson":"https://agentstaffer.com/skills/remoteopenclaw-evaluating-llms-harness/profile.json","installJson":"https://agentstaffer.com/skills/remoteopenclaw-evaluating-llms-harness/install.json","trustJson":"https://agentstaffer.com/skills/remoteopenclaw-evaluating-llms-harness/trust.json","trialJson":"https://agentstaffer.com/skills/remoteopenclaw-evaluating-llms-harness/trial.json"},"queueRank":4,"recommendation":{"label":"Needs review","score":38},"trialPriority":58,"trial":{"title":"Run a cited research brief trial","input":"Ask for a one-page brief on a narrow question with source links and confidence notes.","expectedOutput":"A sourced summary that separates facts, assumptions, and unresolved questions.","safetyNotes":["Run in a sandbox or throwaway workspace.","Use non-production data and temporary credentials.","Capture output, logs, install friction, and any permission requests."],"decisionCriteria":["Cites primary sources","Flags uncertainty","Avoids unsupported claims","Keeps scope tight"],"runUrl":"https://agentstaffer.com/skills/remoteopenclaw-evaluating-llms-harness#trial","runPath":"/skills/remoteopenclaw-evaluating-llms-harness#trial","createEndpoint":"https://agentstaffer.com/api/actions/run-trial","createBody":{"listingId":"remoteopenclaw:skills-hermes-built-in-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}},"launchPacket":{"schemaVersion":"agentstaffer.trial-launch-packet-lite.v1","requiredPermissions":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","License review: License metadata is missing from the current public profile."]},"preflightGates":[{"id":"profile","label":"Profile record","href":"https://agentstaffer.com/skills/remoteopenclaw-evaluating-llms-harness/profile.json","passed":true},{"id":"install","label":"Setup guide","href":"https://agentstaffer.com/skills/remoteopenclaw-evaluating-llms-harness/install.json","passed":true},{"id":"sandbox","label":"Use sandbox data","passed":true}],"promotionGates":[{"id":"install","label":"Install path reviewed","passed":true},{"id":"trust","label":"Trust evidence reviewed","passed":false},{"id":"trial","label":"Sandbox trial recorded","passed":false}],"stopConditions":["Install asks for production credentials or broad write access.","Source or package cannot be verified from profile evidence."],"createRequest":{"endpoint":"https://agentstaffer.com/api/actions/run-trial","method":"POST","body":{"listingId":"remoteopenclaw:skills-hermes-built-in-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}}},"safety":{"trustTier":"Needs review","installReadiness":"Source guide","topRisk":{"level":"medium","label":"No community evaluation","detail":"AgentStaffer users have not submitted a review yet."},"requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required"},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review"}],"permissionSummary":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","License review: License metadata is missing from the current public profile."]}}},{"id":"claudemarketplaces:skills-davila7-claude-code-templates-evaluating-llms-harness","slug":"claudemarketplaces-evaluating-llms-harness","name":"evaluating-llms-harness","type":"skill","source":"claudemarketplaces","status":"pending","summary":"A wrapper around the lm-evaluation-harness library that lets you benchmark language models against 60+ academic tests right from your terminal. You point it at any HuggingFace model and specify which benchmarks to run (MMLU, GSM8k, HellaSwag, etc.), and it handles the standardized prompts and scoring. Useful when you're comparing models or trying to get obj...","profileUrl":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness","fitScore":38,"roleFamily":"Coding & Development","reviewSignal":"Pending","urls":{"profileJson":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/profile.json","installJson":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/install.json","trustJson":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/trust.json","trialJson":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/trial.json"},"queueRank":5,"recommendation":{"label":"Needs review","score":38},"trialPriority":58,"trial":{"title":"Run a small code review trial","input":"Give the agent one low-risk issue or one small pull request and ask for defects, tests, and a patch plan.","expectedOutput":"A concise review with reproducible findings, changed-file scope, and test suggestions.","safetyNotes":["Run in a sandbox or throwaway workspace.","Use non-production data and temporary credentials.","Capture output, logs, install friction, and any permission requests."],"decisionCriteria":["Finds concrete issues","Avoids broad rewrites","Explains tradeoffs","Can be evaluated without secrets"],"runUrl":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness#trial","runPath":"/skills/claudemarketplaces-evaluating-llms-harness#trial","createEndpoint":"https://agentstaffer.com/api/actions/run-trial","createBody":{"listingId":"claudemarketplaces:skills-davila7-claude-code-templates-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}},"launchPacket":{"schemaVersion":"agentstaffer.trial-launch-packet-lite.v1","requiredPermissions":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required","text":"Local terminal: A CLI path is visible, but the exact command must be verified from source."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","Local terminal: A CLI path is visible, but the exact command must be verified from source.","License review: License metadata is missing from the current public profile."]},"preflightGates":[{"id":"profile","label":"Profile record","href":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/profile.json","passed":true},{"id":"install","label":"Setup guide","href":"https://agentstaffer.com/skills/claudemarketplaces-evaluating-llms-harness/install.json","passed":true},{"id":"sandbox","label":"Use sandbox data","passed":true}],"promotionGates":[{"id":"install","label":"Install path reviewed","passed":true},{"id":"trust","label":"Trust evidence reviewed","passed":false},{"id":"trial","label":"Sandbox trial recorded","passed":false}],"stopConditions":["Install asks for production credentials or broad write access.","Source or package cannot be verified from profile evidence."],"createRequest":{"endpoint":"https://agentstaffer.com/api/actions/run-trial","method":"POST","body":{"listingId":"claudemarketplaces:skills-davila7-claude-code-templates-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}}},"safety":{"trustTier":"Needs review","installReadiness":"CLI likely","topRisk":{"level":"medium","label":"No community evaluation","detail":"AgentStaffer users have not submitted a review yet."},"requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required"},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required"},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review"}],"permissionSummary":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"terminal","label":"Local terminal","detail":"A CLI path is visible, but the exact command must be verified from source.","severity":"required","text":"Local terminal: A CLI path is visible, but the exact command must be verified from source."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","Local terminal: A CLI path is visible, but the exact command must be verified from source.","License review: License metadata is missing from the current public profile."]}}},{"id":"smithery:skills-davila7-evaluating-llms-harness","slug":"smithery-evaluating-llms-harness","name":"Evaluating Llms Harness","type":"skill","source":"smithery","status":"pending","summary":"Evaluating Llms Harness is a Smithery skill listing by davila7.","profileUrl":"https://agentstaffer.com/skills/smithery-evaluating-llms-harness","fitScore":24,"roleFamily":"Evaluation & Safety","reviewSignal":"Pending","urls":{"profileJson":"https://agentstaffer.com/skills/smithery-evaluating-llms-harness/profile.json","installJson":"https://agentstaffer.com/skills/smithery-evaluating-llms-harness/install.json","trustJson":"https://agentstaffer.com/skills/smithery-evaluating-llms-harness/trust.json","trialJson":"https://agentstaffer.com/skills/smithery-evaluating-llms-harness/trial.json"},"queueRank":6,"recommendation":{"label":"Needs review","score":24},"trialPriority":44,"trial":{"title":"Run a sandbox capability trial","input":"Ask the agent to complete one narrow task that reflects your intended use, using public or mock data only.","expectedOutput":"A useful result, clear assumptions, visible limitations, and enough evidence for a human evaluation.","safetyNotes":["Run in a sandbox or throwaway workspace.","Use non-production data and temporary credentials.","Capture output, logs, install friction, and any permission requests."],"decisionCriteria":["Completes the narrow task","Explains limits","Requires minimal privileges","Produces reviewable evidence"],"runUrl":"https://agentstaffer.com/skills/smithery-evaluating-llms-harness#trial","runPath":"/skills/smithery-evaluating-llms-harness#trial","createEndpoint":"https://agentstaffer.com/api/actions/run-trial","createBody":{"listingId":"smithery:skills-davila7-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}},"launchPacket":{"schemaVersion":"agentstaffer.trial-launch-packet-lite.v1","requiredPermissions":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","License review: License metadata is missing from the current public profile."]},"preflightGates":[{"id":"profile","label":"Profile record","href":"https://agentstaffer.com/skills/smithery-evaluating-llms-harness/profile.json","passed":true},{"id":"install","label":"Setup guide","href":"https://agentstaffer.com/skills/smithery-evaluating-llms-harness/install.json","passed":true},{"id":"sandbox","label":"Use sandbox data","passed":true}],"promotionGates":[{"id":"install","label":"Install path reviewed","passed":true},{"id":"trust","label":"Trust evidence reviewed","passed":false},{"id":"trial","label":"Sandbox trial recorded","passed":false}],"stopConditions":["Install asks for production credentials or broad write access.","Source or package cannot be verified from profile evidence."],"createRequest":{"endpoint":"https://agentstaffer.com/api/actions/run-trial","method":"POST","body":{"listingId":"smithery:skills-davila7-evaluating-llms-harness","reviewer":"Trial reviewer","notes":"Sandbox run using mock data."}}},"safety":{"trustTier":"Needs review","installReadiness":"Source guide","topRisk":{"level":"medium","label":"No community evaluation","detail":"AgentStaffer users have not submitted a review yet."},"requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required"},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review"}],"permissionSummary":{"schemaVersion":"agentstaffer.trial-requirements.v1","principle":"Use the smallest sandbox permission set needed for the first reversible trial.","requirements":[{"id":"sandbox","label":"Sandbox workspace","detail":"Run the first trial in a throwaway directory or isolated environment.","severity":"required","text":"Sandbox workspace: Run the first trial in a throwaway directory or isolated environment."},{"id":"license-review","label":"License review","detail":"License metadata is missing from the current public profile.","severity":"review","text":"License review: License metadata is missing from the current public profile."}],"humanReadable":["Sandbox workspace: Run the first trial in a throwaway directory or isolated environment.","License review: License metadata is missing from the current public profile."]}}}],"recentRuns":[],"pagination":{"page":1,"pageSize":24,"totalItems":6,"totalPages":1,"nextPageUrl":null,"prevPageUrl":null},"filters":{"q":"evaluating-llms-harness","type":null,"role":null,"protocol":null,"trust":null,"install":null,"source":null,"status":null,"sort":"trial_readiness"},"availableFilters":{"roles":[{"id":"coding-development","label":"Coding & Development"},{"id":"workflow-automation","label":"Workflow Automation"},{"id":"multi-agent-orchestration","label":"Multi-agent Orchestration"},{"id":"mcp-tool-integration","label":"MCP & Tool Integration"},{"id":"research-knowledge","label":"Research & Knowledge"},{"id":"computer-use-desktop","label":"Computer Use & Desktop"},{"id":"business-operations","label":"Business Operations"},{"id":"data-integrations","label":"Data & Integrations"},{"id":"evaluation-safety","label":"Evaluation & Safety"},{"id":"model-inference","label":"Models & Inference"},{"id":"general-agent","label":"General Agent"}],"protocols":[{"value":"cli","label":"CLI (4)"},{"value":"http","label":"HTTP (3)"},{"value":"mcp","label":"MCP (2)"}],"trust":[{"value":"needs-review","label":"Needs review (6)"}],"install":[{"value":"cli-likely","label":"CLI likely (2)"},{"value":"mcp-ready","label":"MCP ready (2)"},{"value":"source-guide","label":"Source guide (2)"}],"sources":[{"value":"claudemarketplaces","label":"claudemarketplaces (2)"},{"value":"explainx","label":"explainx (1)"},{"value":"mcpservers","label":"mcpservers (1)"},{"value":"remoteopenclaw","label":"remoteopenclaw (1)"},{"value":"smithery","label":"smithery (1)"}],"types":[{"value":"skill","label":"skill (6)"}],"sortModes":["relevance","trust","updated","popularity","install_readiness","trial_readiness"]},"launchProtocol":{"schemaVersion":"agentstaffer.trial-launch-packet.v1","location":"trialQueue[].launchPacket","permissionsLocation":"trialQueue[].safety.permissionSummary","rule":"Use launchPacket.preflightGates and launchPacket.stopConditions before calling launchPacket.createRequest.","promotionRule":"Do not promote or hire until launchPacket.promotionGates are clear or explicitly accepted by a human reviewer."},"visibleRetrievalSpine":[{"step":"01","label":"llms.txt","href":"/llms.txt","note":"Site policy and safety context"},{"step":"02","label":"trials.json","href":"/trials.json?q=evaluating-llms-harness&sort=trial_readiness","note":"Action queue and page contract"},{"step":"03","label":"profile HTML","href":"/skills/explainx-evaluating-llms-harness","note":"Human decision summary"},{"step":"04","label":"profile package","href":"/skills/explainx-evaluating-llms-harness/profile.json","note":"Fetch all 9 required finalist resources","requiredResources":["profile.json","score.json","evidence.json","trust.json","trial.json","install.json","history.json","similar.json","dossier.md"]}],"profileResourceContract":{"schemaVersion":"agentstaffer.profile-resource-contract.v1","requiredResources":["profile.json","score.json","evidence.json","trust.json","trial.json","install.json","history.json","similar.json","dossier.md"],"recommendedFetchOrder":["profile.json","score.json","evidence.json","trust.json","trial.json","install.json","history.json","similar.json","dossier.md"]},"retrievalHints":["Use trialQueue[].trial before recommending hire or production installation.","Use trialQueue[].safety.permissionSummary.humanReadable when explaining first-run permissions to a human.","Use trialQueue[].launchPacket as the copyable trial launch contract for humans and AI agents.","POST to /api/actions/run-trial with trialQueue[].trial.createBody to create a public trial result.","Fetch trialQueue[].urls.profileJson (profile.json) and trialQueue[].urls.installJson (install.json) for the full dossier and install payloads.","Use recentRuns[] as auditable evidence of completed or blocked sandbox evaluations."],"links":{"trials":"https://agentstaffer.com/trials?q=evaluating-llms-harness&sort=trial_readiness","trialsJson":"https://agentstaffer.com/trials.json?q=evaluating-llms-harness&sort=trial_readiness","compare":"https://agentstaffer.com/compare","promotions":"https://agentstaffer.com/promotions","jobMatcher":"https://agentstaffer.com/job-matcher","openApi":"https://agentstaffer.com/openapi.json","llmsTxt":"https://agentstaffer.com/llms.txt","geoJson":"https://agentstaffer.com/geo.json","agentVisibilityJson":"https://agentstaffer.com/agent-visibility.json","aiCrawlMonitorJson":"https://agentstaffer.com/ai-crawl-monitor.json"}}