> ## Documentation Index
> Fetch the complete documentation index at: https://www.traceloop.com/docs/llms.txt
> Use this file to discover all available pages before exploring further.

# Create a custom LLM-as-a-judge evaluator

> Create a new custom evaluator at the organization level without project or environment bindings. The evaluator uses LLM-as-a-judge to evaluate inputs based on the provided configuration. The response format is automatically generated from the output schema.



## OpenAPI

````yaml post /v2/evaluators
openapi: 3.0.0
info:
  title: Traceloop API
  version: 1.0.0
  contact: {}
servers:
  - url: https://api.traceloop.com
security: []
paths:
  /v2/evaluators:
    post:
      tags:
        - evaluators
      summary: Create a custom LLM-as-a-judge evaluator
      description: >-
        Create a new custom evaluator at the organization level without project
        or environment bindings. The evaluator uses LLM-as-a-judge to evaluate
        inputs based on the provided configuration. The response format is
        automatically generated from the output schema.
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.CreateCustomEvaluatorInput'
        description: Custom evaluator creation input
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.CreateEvaluatorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
components:
  schemas:
    request.CreateCustomEvaluatorInput:
      properties:
        description:
          type: string
        frequency_penalty:
          type: number
        input_schema:
          description: Schemas
          items:
            $ref: '#/components/schemas/evaluator.Property'
          type: array
        logprobs:
          type: boolean
        max_tokens:
          type: integer
        messages:
          description: LLM Message configuration
          items:
            $ref: '#/components/schemas/model.LLMMessage'
          type: array
        mode:
          type: string
        model:
          description: LLM parameters (from BaseLLMConfig)
          type: string
        name:
          description: Basic fields
          type: string
        output_schema:
          items:
            $ref: '#/components/schemas/evaluator.Property'
          type: array
        presence_penalty:
          type: number
        provider:
          description: Provider configuration
          type: string
        slug:
          type: string
        stop:
          items:
            type: string
          type: array
        temperature:
          type: number
        top_p:
          type: number
      required:
        - input_schema
        - messages
        - model
        - name
        - output_schema
        - provider
      type: object
    response.CreateEvaluatorResponse:
      properties:
        bindings:
          items:
            type: string
          type: array
        evaluator_id:
          type: string
        slug:
          type: string
        type:
          $ref: '#/components/schemas/types.EvaluatorType'
        version_id:
          type: string
      type: object
    response.ErrorResponse:
      description: Standard error response structure
      properties:
        error:
          example: error message
          type: string
      type: object
    evaluator.Property:
      properties:
        description:
          type: string
        enum_values:
          description: >-
            The allowed values for this property. Only applicable when `type` is
            `enum`.
          items:
            type: string
          type: array
        label:
          type: string
        name:
          type: string
        type:
          $ref: '#/components/schemas/evaluator.PropertyType'
      required:
        - name
        - type
      type: object
    model.LLMMessage:
      properties:
        content:
          type: string
        role:
          type: string
        tool_call_id:
          type: string
      type: object
    types.EvaluatorType:
      enum:
        - AGENT_GOAL_ACCURACY
        - CHAR_COUNT
        - CHAR_COUNT_RATIO
        - ANSWER_COMPLETENESS
        - FAITHFULNESS
        - JSON_VALIDATOR
        - PERPLEXITY
        - PII_DETECTOR
        - PLACEHOLDER_REGEX
        - PROFANITY_DETECTOR
        - SEXISM_DETECTOR
        - PROMPT_INJECTION
        - TONE_DETECTION
        - PROMPT_PERPLEXITY
        - TOXICITY_DETECTOR
        - ANSWER_RELEVANCY
        - ANSWER_CORRECTNESS
        - REGEX_VALIDATOR
        - SECRETS_DETECTOR
        - SQL_VALIDATOR
        - SEMANTIC_SIMILARITY
        - TOPIC_ADHERENCE
        - UNCERTAINTY_DETECTOR
        - WORD_COUNT
        - WORD_COUNT_RATIO
        - INSTRUCTION_ADHERENCE
        - AGENT_FLOW_QUALITY
        - AGENT_EFFICIENCY
        - AGENT_GOAL_COMPLETENESS
        - CONVERSATION_QUALITY
        - INTENT_CHANGE
        - CONTEXT_RELEVANCE
        - AGENT_TOOL_ERROR_DETECTOR
        - AGENT_TOOL_TRAJECTORY
        - HTML_COMPARISON
        - LLM_AS_A_JUDGE
      type: string
      x-enum-varnames:
        - AgentGoalAccuracy
        - CharCount
        - CharCountRatio
        - AnswerCompleteness
        - Faithfulness
        - JsonValidator
        - Perplexity
        - PIIDetector
        - PlaceholderRegex
        - ProfanityDetector
        - SexismDetector
        - PromptInjection
        - ToneDetection
        - PromptPerplexity
        - ToxicityDetector
        - AnswerRelevancy
        - AnswerCorrectness
        - RegexValidator
        - SecretsDetector
        - SQLValidator
        - SemanticSimilarity
        - TopicAdherence
        - UncertaintyDetector
        - WordCount
        - WordCountRatio
        - InstructionAdherence
        - AgentFlowQuality
        - AgentEfficiency
        - AgentGoalCompleteness
        - ConversationQuality
        - IdentIntentChange
        - ContextRelevance
        - AgentToolErrorDetector
        - AgentToolTrajectory
        - HtmlComparison
        - LLMAsAJudge
    evaluator.PropertyType:
      enum:
        - string
        - int
        - float
        - boolean
        - enum
      type: string
      x-enum-varnames:
        - StringType
        - IntType
        - FloatType
        - BooleanType
        - EnumType

````