# Evaluation

Types:

```python
from nemo_platform.types.evaluation import (
    Agent,
    AgentGoalAccuracyMetric,
    AgentGoalAccuracyMetricParam,
    AggregatedMetricResult,
    AnswerAccuracyMetric,
    AnswerAccuracyMetricParam,
    BleuMetric,
    BleuMetricParam,
    ContextEntityRecallMetric,
    ContextEntityRecallMetricParam,
    ContextPrecisionMetric,
    ContextPrecisionMetricParam,
    ContextRecallMetric,
    ContextRecallMetricParam,
    ContextRelevanceMetric,
    ContextRelevanceMetricParam,
    DatasetRows,
    EvaluationOnlineModelParams,
    EvaluationOnlineParams,
    EvaluationParams,
    ExactMatchMetric,
    ExactMatchMetricParam,
    F1Metric,
    F1MetricParam,
    FaithfulnessMetric,
    FaithfulnessMetricParam,
    FieldMapping,
    FilesetRef,
    InferenceParams,
    JsonScoreParser,
    LLMJudgeMetric,
    LLMJudgeMetricParam,
    MetricRef,
    MetricScore,
    Model,
    NeMoAgentToolkitRemoteMetric,
    NeMoAgentToolkitRemoteMetricParam,
    NoiseSensitivityMetric,
    NoiseSensitivityMetricParam,
    NumberCheckMetric,
    NumberCheckMetricParam,
    Parameter,
    RangeScore,
    ReasoningParams,
    RegexScoreParser,
    RemoteMetric,
    RemoteMetricParam,
    RemoteScore,
    ResponseGroundednessMetric,
    ResponseGroundednessMetricParam,
    ResponseRelevancyMetric,
    ResponseRelevancyMetricParam,
    RougeMetric,
    RougeMetricParam,
    RowScore,
    Rubric,
    RubricScore,
    ScoreStats,
    StringCheckMetric,
    StringCheckMetricParam,
    ToolCallAccuracyMetric,
    ToolCallAccuracyMetricParam,
    ToolCallingMetric,
    ToolCallingMetricParam,
    TopicAdherenceMetric,
    TopicAdherenceMetricParam,
)
```

## Benchmarks

Types:

```python
from nemo_platform.types.evaluation import (
    Benchmark,
    BenchmarkRequest,
    BenchmarksListResponse,
    ExtendedBenchmark,
    SystemBenchmark,
    SystemMetric,
    BenchmarkCreateResponse,
    BenchmarkRetrieveResponse,
)
```

Methods:

- client.evaluation.benchmarks.create(\*, workspace, \*\*params) -> BenchmarkCreateResponse
- client.evaluation.benchmarks.retrieve(name, \*, workspace, \*\*params) -> BenchmarkRetrieveResponse
- client.evaluation.benchmarks.list(\*, workspace, \*\*params) -> SyncDefaultPagination[Data]
- client.evaluation.benchmarks.delete(name, \*, workspace) -> DeleteResponse

## BenchmarkJobs

Types:

```python
from nemo_platform.types.evaluation import (
    BenchmarkEvaluationJob,
    BenchmarkEvaluationJobRequest,
    BenchmarkEvaluationJobsListFilter,
    BenchmarkEvaluationJobsPage,
    BenchmarkEvaluationJobsSortField,
    BenchmarkOfflineJob,
    BenchmarkOnlineAgentJob,
    BenchmarkOnlineJob,
    BenchmarkRef,
    SystemBenchmarkOfflineJob,
    SystemBenchmarkOnlineJob,
)
```

Methods:

- client.evaluation.benchmark_jobs.create(\*, workspace, \*\*params) -> BenchmarkEvaluationJob
- client.evaluation.benchmark_jobs.retrieve(name, \*, workspace) -> BenchmarkEvaluationJob
- client.evaluation.benchmark_jobs.list(\*, workspace, \*\*params) -> SyncDefaultPagination[BenchmarkEvaluationJob]
- client.evaluation.benchmark_jobs.delete(name, \*, workspace) -> None
- client.evaluation.benchmark_jobs.cancel(name, \*, workspace) -> BenchmarkEvaluationJob
- client.evaluation.benchmark_jobs.get_logs(name, \*, workspace, \*\*params) -> SyncLogsPagination[PlatformJobLog]
- client.evaluation.benchmark_jobs.get_status(name, \*, workspace) -> PlatformJobStatusResponse

### Results

Methods:

- client.evaluation.benchmark_jobs.results.retrieve(name, \*, workspace, job) -> PlatformJobResultResponse
- client.evaluation.benchmark_jobs.results.list(name, \*, workspace) -> PlatformJobListResultResponse
- client.evaluation.benchmark_jobs.results.download(name, \*, workspace, job) -> BinaryAPIResponse

#### AggregateScores

Types:

```python
from nemo_platform.types.evaluation.benchmark_jobs.results import (
    BenchmarkEvaluationResult,
    BenchmarkMetricResult,
)
```

Methods:

- client.evaluation.benchmark_jobs.results.aggregate_scores.download(job, \*, workspace) -> BenchmarkEvaluationResult

#### RowScores

Methods:

- client.evaluation.benchmark_jobs.results.row_scores.download(job, \*, workspace, \*\*params) -> JSONLDecoder[RowScore]

#### Artifacts

Methods:

- client.evaluation.benchmark_jobs.results.artifacts.download(job, \*, workspace) -> BinaryAPIResponse

## BenchmarkJobResults

Types:

```python
from nemo_platform.types.evaluation import BenchmarkJobResult, BenchmarkJobResultsListResponse
```

Methods:

- client.evaluation.benchmark_job_results.retrieve(name, \*, workspace, \*\*params) -> BenchmarkJobResult
- client.evaluation.benchmark_job_results.list(\*, workspace, \*\*params) -> SyncDefaultPagination[BenchmarkJobResult]
- client.evaluation.benchmark_job_results.delete(name, \*, workspace) -> DeleteResponse

## Metrics

Types:

```python
from nemo_platform.types.evaluation import (
    AgentGoalAccuracyMetricResponse,
    AggregateRangeScore,
    AggregateRubricScore,
    AnswerAccuracyMetricResponse,
    BleuMetricResponse,
    ContextEntityRecallMetricResponse,
    ContextPrecisionMetricResponse,
    ContextRecallMetricResponse,
    ContextRelevanceMetricResponse,
    EvaluateDatasetRows,
    ExactMatchMetricResponse,
    F1MetricResponse,
    FaithfulnessMetricResponse,
    Histogram,
    HistogramBin,
    LLMJudgeMetricResponse,
    MetricEvaluationRequest,
    MetricEvaluationResponse,
    MetricEvaluationRowScore,
    MetricType,
    MetricsListResponse,
    ModelRef,
    NeMoAgentToolkitRemoteMetricResponse,
    NoiseSensitivityMetricResponse,
    NumberCheckMetricResponse,
    Percentiles,
    RemoteMetricResponse,
    ResponseGroundednessMetricResponse,
    ResponseRelevancyMetricResponse,
    RougeMetricResponse,
    RubricScoreStat,
    StringCheckMetricResponse,
    SystemMetricResponse,
    ToolCallAccuracyMetricResponse,
    ToolCallingMetricResponse,
    TopicAdherenceMetricResponse,
    MetricCreateResponse,
    MetricRetrieveResponse,
)
```

Methods:

- client.evaluation.metrics.create(name, \*, workspace, \*\*params) -> MetricCreateResponse
- client.evaluation.metrics.retrieve(name, \*, workspace) -> MetricRetrieveResponse
- client.evaluation.metrics.list(\*, workspace, \*\*params) -> SyncDefaultPagination[Data]
- client.evaluation.metrics.delete(name, \*, workspace) -> DeleteResponse
- client.evaluation.metrics.evaluate(\*, workspace, \*\*params) -> MetricEvaluationResponse

## MetricJobs

Types:

```python
from nemo_platform.types.evaluation import (
    BuiltInDataset,
    Fileset,
    MetricEvaluationJob,
    MetricEvaluationJobRequest,
    MetricEvaluationJobsListFilter,
    MetricEvaluationJobsPage,
    MetricEvaluationJobsSortField,
    MetricOfflineJob,
    MetricOnlineAgentJob,
    MetricOnlineJob,
    MetricRetrieverJob,
    RetrieverPipeline,
    SystemMetricParam,
)
```

Methods:

- client.evaluation.metric_jobs.create(\*, workspace, \*\*params) -> MetricEvaluationJob
- client.evaluation.metric_jobs.retrieve(name, \*, workspace) -> MetricEvaluationJob
- client.evaluation.metric_jobs.list(\*, workspace, \*\*params) -> SyncDefaultPagination[MetricEvaluationJob]
- client.evaluation.metric_jobs.delete(name, \*, workspace) -> None
- client.evaluation.metric_jobs.cancel(name, \*, workspace) -> MetricEvaluationJob
- client.evaluation.metric_jobs.get_logs(name, \*, workspace, \*\*params) -> SyncLogsPagination[PlatformJobLog]
- client.evaluation.metric_jobs.get_status(name, \*, workspace) -> PlatformJobStatusResponse

### Results

Methods:

- client.evaluation.metric_jobs.results.retrieve(name, \*, workspace, job) -> PlatformJobResultResponse
- client.evaluation.metric_jobs.results.list(name, \*, workspace) -> PlatformJobListResultResponse
- client.evaluation.metric_jobs.results.download(name, \*, workspace, job) -> BinaryAPIResponse

#### AggregateScores

Methods:

- client.evaluation.metric_jobs.results.aggregate_scores.download(job, \*, workspace) -> AggregatedMetricResult

#### RowScores

Methods:

- client.evaluation.metric_jobs.results.row_scores.download(job, \*, workspace, \*\*params) -> JSONLDecoder[RowScore]

#### Artifacts

Methods:

- client.evaluation.metric_jobs.results.artifacts.download(job, \*, workspace) -> BinaryAPIResponse

## MetricJobResults

Types:

```python
from nemo_platform.types.evaluation import MetricJobResult, MetricJobResultsListResponse
```

Methods:

- client.evaluation.metric_job_results.retrieve(name, \*, workspace, \*\*params) -> MetricJobResult
- client.evaluation.metric_job_results.list(\*, workspace, \*\*params) -> SyncDefaultPagination[MetricJobResult]
- client.evaluation.metric_job_results.delete(name, \*, workspace) -> DeleteResponse
