How to change the target before automatic calling the scoring function

[animeshkumarpaul]

EpochScoring_cb_train = EpochScoring(scoring=f1_custom, lower_is_better=False, on_train = True, target_extractor= target_extractor_custom, name = 'f1_train')

monitor = lambda net: all(net.history[-1, (
     'train_loss_best', 'valid_loss_best')])

checkpoint_cb = Checkpoint(monitor=monitor, f_params="params_{last_epoch[epoch]}.pt", load_best=True, dirname='Logs')

device='cuda:5'

net = NeuralNetClassifier(
    ClassifierModule,
    module__layer_sizes= [...,...,...],
    module__input_size=X.shape[1],
    max_epochs=200,
    batch_size=50000,
    device= device,
   
    criterion=nn.CrossEntropyLoss,
    criterion__reduction='mean',
    criterion__device= device, 

    optimizer=torch.optim.Adam,
    lr=0.01,
    optimizer__weight_decay=1e-4,
   
    
    callbacks=[EpochScoring_cb_train],
    train_split=predefined_split(valid_ds)
)

I am getting this error maybe because my class label is continuous.

 ----> 3 net.fit(X_train_NN,y_train)

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/skorch/classifier.py:143, in NeuralNetClassifier.fit(self, X, y, **fit_params)
    132 """See ``NeuralNet.fit``.
    133 
    134 In contrast to ``NeuralNet.fit``, ``y`` is non-optional to
   (...)
    138 
    139 """
    140 # pylint: disable=useless-super-delegation
    141 # this is actually a pylint bug:
    142 # https://github.com/PyCQA/pylint/issues/1085
--> 143 return super(NeuralNetClassifier, self).fit(X, y, **fit_params)

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/skorch/net.py:1302, in NeuralNet.fit(self, X, y, **fit_params)
   1299 if not self.warm_start or not self.initialized_:
   1300     self.initialize()
-> 1302 self.partial_fit(X, y, **fit_params)
   1303 return self

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/skorch/net.py:1261, in NeuralNet.partial_fit(self, X, y, classes, **fit_params)
   1259 self.notify('on_train_begin', X=X, y=y)
   1260 try:
-> 1261     self.fit_loop(X, y, **fit_params)
   1262 except KeyboardInterrupt:
   1263     pass

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/skorch/net.py:1179, in NeuralNet.fit_loop(self, X, y, epochs, **fit_params)
   1173     self.run_single_epoch(iterator_train, training=True, prefix="train",
   1174                           step_fn=self.train_step, **fit_params)
   1176     self.run_single_epoch(iterator_valid, training=False, prefix="valid",
   1177                           step_fn=self.validation_step, **fit_params)
-> 1179     self.notify("on_epoch_end", **on_epoch_kwargs)
   1180 return self

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/skorch/net.py:369, in NeuralNet.notify(self, method_name, **cb_kwargs)
    367 getattr(self, method_name)(self, **cb_kwargs)
    368 for _, cb in self.callbacks_:
--> 369     getattr(cb, method_name)(self, **cb_kwargs)

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/skorch/callbacks/scoring.py:467, in EpochScoring.on_epoch_end(self, net, dataset_train, dataset_valid, **kwargs)
    464     return
    466 with _cache_net_forward_iter(net, self.use_caching, y_pred) as cached_net:
--> 467     current_score = self._scoring(cached_net, X_test, y_test)
    469 self._record_score(net.history, current_score)

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/skorch/callbacks/scoring.py:175, in ScoringBase._scoring(self, net, X_test, y_test)
    172 """Resolve scoring and apply it to data. Use cached prediction
    173 instead of running inference again, if available."""
    174 scorer = check_scoring(net, self.scoring_)
--> 175 return scorer(net, X_test, y_test)

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/sklearn/metrics/_scorer.py:220, in _BaseScorer.__call__(self, estimator, X, y_true, sample_weight)
    197 def __call__(self, estimator, X, y_true, sample_weight=None):
    198     """Evaluate predicted target values for X relative to y_true.
    199 
    200     Parameters
   (...)
    218         Score function applied to prediction of estimator on X.
    219     """
--> 220     return self._score(
    221         partial(_cached_call, None),
    222         estimator,
    223         X,
    224         y_true,
    225         sample_weight=sample_weight,
    226     )

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/sklearn/metrics/_scorer.py:268, in _PredictScorer._score(self, method_caller, estimator, X, y_true, sample_weight)
    264     return self._sign * self._score_func(
    265         y_true, y_pred, sample_weight=sample_weight, **self._kwargs
    266     )
    267 else:
--> 268     return self._sign * self._score_func(y_true, y_pred, **self._kwargs)

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/sklearn/utils/_param_validation.py:192, in validate_params.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
    187 validate_parameter_constraints(
    188     parameter_constraints, params, caller_name=func.__qualname__
    189 )
    191 try:
--> 192     return func(*args, **kwargs)
    193 except InvalidParameterError as e:
    194     # When the function is just a wrapper around an estimator, we allow
    195     # the function to delegate validation to the estimator, but we replace
    196     # the name of the estimator by the name of the function in the error
    197     # message to avoid confusion.
    198     msg = re.sub(
    199         r"parameter of \w+ must be",
    200         f"parameter of {func.__qualname__} must be",
    201         str(e),
    202     )

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/sklearn/metrics/_classification.py:221, in accuracy_score(y_true, y_pred, normalize, sample_weight)
    155 """Accuracy classification score.
    156 
    157 In multilabel classification, this function computes subset accuracy:
   (...)
    217 0.5
    218 """
    220 # Compute accuracy for each possible representation
--> 221 y_type, y_true, y_pred = _check_targets(y_true, y_pred)
    222 check_consistent_length(y_true, y_pred, sample_weight)
    223 if y_type.startswith("multilabel"):

File ~/anaconda3/envs/xgb3_ray2_pytorch/lib/python3.8/site-packages/sklearn/metrics/_classification.py:95, in _check_targets(y_true, y_pred)
     92     y_type = {"multiclass"}
     94 if len(y_type) > 1:
---> 95     raise ValueError(
     96         "Classification metrics can't handle a mix of {0} and {1} targets".format(
     97             type_true, type_pred
     98         )
     99     )
    101 # We can't have more than one value on y_type => The set is no more needed
    102 y_type = y_type.pop()

ValueError: Classification metrics can't handle a mix of continuous and binary targets

My questions:

• How can I change the target before calling the scoring function?
• I think the scoring function by default calls the valid accuracy which is generating the errors. And, I don't want to use accuracy for evaluation. So how to stop this default call?

[BenjaminBossan]

I am getting this error maybe because my class label is continuous.

This probably means that you're not dealing with a classification problem. Try using NeuralNet instead of NeuralNetClassifier.