Skip to content

Commit 8a9f1f2

Browse files
daholstedaholste
authored
API rev (dotnet#181)
1 parent 51a613c commit 8a9f1f2

14 files changed

Lines changed: 159 additions & 100 deletions

‎src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,13 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
5454
_settings = settings;
5555
}
5656

57-
public IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizers = null)
57+
public IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizers = null)
58+
{
59+
var columnInformation = new ColumnInformation() { LabelColumn = labelColumn };
60+
return Execute(_context, trainData, columnInformation, null, preFeaturizers);
61+
}
62+
63+
public IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizers = null)
5864
{
5965
return Execute(_context, trainData, columnInformation, null, preFeaturizers);
6066
}
@@ -81,7 +87,7 @@ internal IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(MLContext c
8187
// run autofit & get all pipelines run in that process
8288
var autoFitter = new AutoFitter<BinaryClassificationMetrics>(context, TaskKind.BinaryClassification, trainData, columnInfo,
8389
validationData, preFeaturizers, new OptimizingMetricInfo(_settings.OptimizingMetric), _settings.ProgressCallback,
84-
_settings, new BinaryDataScorer(_settings.OptimizingMetric),
90+
_settings, new BinaryMetricsAgent(_settings.OptimizingMetric),
8591
TrainerExtensionUtil.GetTrainerNames(_settings.WhitelistedTrainers));
8692

8793
return autoFitter.Fit();
@@ -90,10 +96,11 @@ internal IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(MLContext c
9096

9197
public static class BinaryExperimentResultExtensions
9298
{
93-
public static RunResult<BinaryClassificationMetrics> Best(this IEnumerable<RunResult<BinaryClassificationMetrics>> results)
99+
public static RunResult<BinaryClassificationMetrics> Best(this IEnumerable<RunResult<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
94100
{
95-
double maxScore = results.Select(r => r.Metrics.Accuracy).Max();
96-
return results.First(r => r.Metrics.Accuracy == maxScore);
101+
var metricsAgent = new BinaryMetricsAgent(metric);
102+
double maxScore = results.Select(r => metricsAgent.GetScore(r.Metrics)).Max();
103+
return results.First(r => metricsAgent.GetScore(r.Metrics) == maxScore);
97104
}
98105
}
99106
}

‎src/Microsoft.ML.Auto/API/MulticlassClassificationExperiment.cs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,13 @@ internal MulticlassClassificationExperiment(MLContext context, MulticlassExperim
5252
_settings = settings;
5353
}
5454

55-
public IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizers = null)
55+
public IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizers = null)
56+
{
57+
var columnInformation = new ColumnInformation() { LabelColumn = labelColumn };
58+
return Execute(_context, trainData, columnInformation, null, preFeaturizers);
59+
}
60+
61+
public IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizers = null)
5662
{
5763
return Execute(_context, trainData, columnInformation, null, preFeaturizers);
5864
}
@@ -79,7 +85,7 @@ internal IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(MLContext c
7985
// run autofit & get all pipelines run in that process
8086
var autoFitter = new AutoFitter<MultiClassClassifierMetrics>(context, TaskKind.MulticlassClassification, trainData,
8187
columnInfo, validationData, preFeaturizers, new OptimizingMetricInfo(_settings.OptimizingMetric),
82-
_settings.ProgressCallback, _settings, new MultiDataScorer(_settings.OptimizingMetric),
88+
_settings.ProgressCallback, _settings, new MultiMetricsAgent(_settings.OptimizingMetric),
8389
TrainerExtensionUtil.GetTrainerNames(_settings.WhitelistedTrainers));
8490

8591
return autoFitter.Fit();
@@ -88,10 +94,11 @@ internal IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(MLContext c
8894

8995
public static class MulticlassExperimentResultExtensions
9096
{
91-
public static RunResult<MultiClassClassifierMetrics> Best(this IEnumerable<RunResult<MultiClassClassifierMetrics>> results)
97+
public static RunResult<MultiClassClassifierMetrics> Best(this IEnumerable<RunResult<MultiClassClassifierMetrics>> results, MulticlassClassificationMetric metric = MulticlassClassificationMetric.AccuracyMicro)
9298
{
93-
double maxScore = results.Select(r => r.Metrics.AccuracyMicro).Max();
94-
return results.First(r => r.Metrics.AccuracyMicro == maxScore);
99+
var metricsAgent = new MultiMetricsAgent(metric);
100+
double maxScore = results.Select(r => metricsAgent.GetScore(r.Metrics)).Max();
101+
return results.First(r => metricsAgent.GetScore(r.Metrics) == maxScore);
95102
}
96103
}
97104
}

‎src/Microsoft.ML.Auto/API/RegressionExperiment.cs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,13 @@ internal RegressionExperiment(MLContext context, RegressionExperimentSettings se
4949
_settings = settings;
5050
}
5151

52-
public IEnumerable<RunResult<RegressionMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizers = null)
52+
public IEnumerable<RunResult<RegressionMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizers = null)
53+
{
54+
var columnInformation = new ColumnInformation() { LabelColumn = labelColumn };
55+
return Execute(_context, trainData, columnInformation, null, preFeaturizers);
56+
}
57+
58+
public IEnumerable<RunResult<RegressionMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizers = null)
5359
{
5460
return Execute(_context, trainData, columnInformation, null, preFeaturizers);
5561
}
@@ -76,7 +82,7 @@ internal IEnumerable<RunResult<RegressionMetrics>> Execute(MLContext context,
7682
// run autofit & get all pipelines run in that process
7783
var autoFitter = new AutoFitter<RegressionMetrics>(context, TaskKind.Regression, trainData, columnInfo,
7884
validationData, preFeaturizers, new OptimizingMetricInfo(_settings.OptimizingMetric),
79-
_settings.ProgressCallback, _settings, new RegressionDataScorer(_settings.OptimizingMetric),
85+
_settings.ProgressCallback, _settings, new RegressionMetricsAgent(_settings.OptimizingMetric),
8086
TrainerExtensionUtil.GetTrainerNames(_settings.WhitelistedTrainers));
8187

8288
return autoFitter.Fit();
@@ -85,10 +91,11 @@ internal IEnumerable<RunResult<RegressionMetrics>> Execute(MLContext context,
8591

8692
public static class RegressionExperimentResultExtensions
8793
{
88-
public static RunResult<RegressionMetrics> Best(this IEnumerable<RunResult<RegressionMetrics>> results)
94+
public static RunResult<RegressionMetrics> Best(this IEnumerable<RunResult<RegressionMetrics>> results, RegressionMetric metric = RegressionMetric.RSquared)
8995
{
90-
double maxScore = results.Select(r => r.Metrics.RSquared).Max();
91-
return results.First(r => r.Metrics.RSquared == maxScore);
96+
var metricsAgent = new RegressionMetricsAgent(metric);
97+
double maxScore = results.Select(r => metricsAgent.GetScore(r.Metrics)).Max();
98+
return results.First(r => metricsAgent.GetScore(r.Metrics) == maxScore);
9299
}
93100
}
94101
}

‎src/Microsoft.ML.Auto/AutoFitter/AutoFitter.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ internal class AutoFitter<T> where T : class
2121
private readonly IEstimator<ITransformer> _preFeaturizers;
2222
private readonly IProgress<RunResult<T>> _progressCallback;
2323
private readonly ExperimentSettings _experimentSettings;
24-
private readonly IDataScorer<T> _dataScorer;
24+
private readonly IMetricsAgent<T> _metricsAgent;
2525
private readonly IEnumerable<TrainerName> _trainerWhitelist;
2626

2727
private IDataView _trainData;
@@ -38,7 +38,7 @@ public AutoFitter(MLContext context,
3838
OptimizingMetricInfo metricInfo,
3939
IProgress<RunResult<T>> progressCallback,
4040
ExperimentSettings experimentSettings,
41-
IDataScorer<T> dataScorer,
41+
IMetricsAgent<T> metricsAgent,
4242
IEnumerable<TrainerName> trainerWhitelist)
4343
{
4444
if (validationData == null)
@@ -56,7 +56,7 @@ public AutoFitter(MLContext context,
5656
_preFeaturizers = preFeaturizers;
5757
_progressCallback = progressCallback;
5858
_experimentSettings = experimentSettings;
59-
_dataScorer = dataScorer;
59+
_metricsAgent = metricsAgent;
6060
_trainerWhitelist = trainerWhitelist;
6161
}
6262

@@ -149,7 +149,7 @@ private SuggestedPipelineResult<T> ProcessPipeline(SuggestedPipeline pipeline)
149149
var pipelineModel = pipeline.Fit(_trainData);
150150
var scoredValidationData = pipelineModel.Transform(_validationData);
151151
var metrics = GetEvaluatedMetrics(scoredValidationData);
152-
var score = _dataScorer.GetScore(metrics);
152+
var score = _metricsAgent.GetScore(metrics);
153153
runResult = new SuggestedPipelineResult<T>(metrics, pipelineModel, pipeline, score, null);
154154
}
155155
catch(Exception ex)

src/Microsoft.ML.Auto/AutoFitter/DataScorer/BinaryDataScorer.cs renamed to src/Microsoft.ML.Auto/AutoFitter/MetricsAgents/BinaryMetricsAgent.cs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@
77

88
namespace Microsoft.ML.Auto
99
{
10-
internal class BinaryDataScorer : IDataScorer<BinaryClassificationMetrics>
10+
internal class BinaryMetricsAgent : IMetricsAgent<BinaryClassificationMetrics>
1111
{
12-
private readonly BinaryClassificationMetric _metric;
12+
private readonly BinaryClassificationMetric _optimizingMetric;
1313

14-
public BinaryDataScorer(BinaryClassificationMetric metric)
14+
public BinaryMetricsAgent(BinaryClassificationMetric optimizingMetric)
1515
{
16-
this._metric = metric;
16+
this._optimizingMetric = optimizingMetric;
1717
}
1818

1919
public double GetScore(BinaryClassificationMetrics metrics)
2020
{
21-
switch(_metric)
21+
switch(_optimizingMetric)
2222
{
2323
case BinaryClassificationMetric.Accuracy:
2424
return metrics.Accuracy;
@@ -39,7 +39,7 @@ public double GetScore(BinaryClassificationMetrics metrics)
3939
}
4040

4141
// never expected to reach here
42-
throw new NotSupportedException($"{_metric} is not a supported sweep metric");
42+
throw new NotSupportedException($"{_optimizingMetric} is not a supported sweep metric");
4343
}
4444
}
4545
}

src/Microsoft.ML.Auto/AutoFitter/DataScorer/IDataScorer.cs renamed to src/Microsoft.ML.Auto/AutoFitter/MetricsAgents/IMetricsAgent.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
namespace Microsoft.ML.Auto
66
{
7-
internal interface IDataScorer<T>
7+
internal interface IMetricsAgent<T>
88
{
99
double GetScore(T metrics);
1010
}

src/Microsoft.ML.Auto/AutoFitter/DataScorer/MultiDataScorer.cs renamed to src/Microsoft.ML.Auto/AutoFitter/MetricsAgents/MultiMetricsAgent.cs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@
77

88
namespace Microsoft.ML.Auto
99
{
10-
internal class MultiDataScorer : IDataScorer<MultiClassClassifierMetrics>
10+
internal class MultiMetricsAgent : IMetricsAgent<MultiClassClassifierMetrics>
1111
{
12-
private readonly MulticlassClassificationMetric _metric;
12+
private readonly MulticlassClassificationMetric _optimizingMetric;
1313

14-
public MultiDataScorer(MulticlassClassificationMetric metric)
14+
public MultiMetricsAgent(MulticlassClassificationMetric optimizingMetric)
1515
{
16-
this._metric = metric;
16+
this._optimizingMetric = optimizingMetric;
1717
}
1818

1919
public double GetScore(MultiClassClassifierMetrics metrics)
2020
{
21-
switch (_metric)
21+
switch (_optimizingMetric)
2222
{
2323
case MulticlassClassificationMetric.AccuracyMacro:
2424
return metrics.AccuracyMacro;
@@ -33,7 +33,7 @@ public double GetScore(MultiClassClassifierMetrics metrics)
3333
}
3434

3535
// never expected to reach here
36-
throw new NotSupportedException($"{_metric} is not a supported sweep metric");
36+
throw new NotSupportedException($"{_optimizingMetric} is not a supported sweep metric");
3737
}
3838
}
3939
}

src/Microsoft.ML.Auto/AutoFitter/DataScorer/RegressionDataScorer.cs renamed to src/Microsoft.ML.Auto/AutoFitter/MetricsAgents/RegressionMetricsAgent.cs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@
77

88
namespace Microsoft.ML.Auto
99
{
10-
internal class RegressionDataScorer : IDataScorer<RegressionMetrics>
10+
internal class RegressionMetricsAgent : IMetricsAgent<RegressionMetrics>
1111
{
12-
private readonly RegressionMetric _metric;
12+
private readonly RegressionMetric _optimizingMetric;
1313

14-
public RegressionDataScorer(RegressionMetric metric)
14+
public RegressionMetricsAgent(RegressionMetric optimizingMetric)
1515
{
16-
this._metric = metric;
16+
this._optimizingMetric = optimizingMetric;
1717
}
1818

1919
public double GetScore(RegressionMetrics metrics)
2020
{
21-
switch(_metric)
21+
switch(_optimizingMetric)
2222
{
2323
case RegressionMetric.L1:
2424
return metrics.L1;
@@ -31,7 +31,7 @@ public double GetScore(RegressionMetrics metrics)
3131
}
3232

3333
// never expected to reach here
34-
throw new NotSupportedException($"{_metric} is not a supported sweep metric");
34+
throw new NotSupportedException($"{_optimizingMetric} is not a supported sweep metric");
3535
}
3636
}
3737
}

‎src/Samples/AutoTrainBinaryClassification.cs‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,8 @@ public static void Run()
3636
// STEP 3: Auto featurize, auto train and auto hyperparameter tuning
3737
Console.WriteLine($"Invoking BinaryClassification.AutoFit");
3838
var autoFitResults = mlContext.AutoInference()
39-
.CreateBinaryClassificationExperiment(new BinaryExperimentSettings()
40-
{
41-
MaxInferenceTimeInSeconds = 60,
42-
OptimizingMetric = BinaryClassificationMetric.Auc
43-
})
44-
.Execute(trainDataView, new ColumnInformation()
45-
{
46-
LabelColumn = LabelColumnName
47-
});
39+
.CreateBinaryClassificationExperiment(60)
40+
.Execute(trainDataView, LabelColumnName);
4841

4942
// STEP 4: Print metric from the best model
5043
var best = autoFitResults.Best();

‎src/Samples/AutoTrainRegression.cs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,8 @@ public static void Run()
3636
// STEP 3: Auto featurize, auto train and auto hyperparameter tuning
3737
Console.WriteLine($"Invoking Regression.AutoFit");
3838
var autoFitResults = mlContext.AutoInference()
39-
.CreateRegressionExperiment(60)
40-
.Execute(trainDataView, new ColumnInformation() {
41-
LabelColumn = LabelColumnName
42-
});
39+
.CreateRegressionExperiment(0)
40+
.Execute(trainDataView, LabelColumnName);
4341

4442
// STEP 4: Compare and print actual value vs predicted value for top 5 rows from validation data
4543
var best = autoFitResults.Best();

0 commit comments

Comments
 (0)