Skip to content

Publish, performance monitor, and score test model versions #217

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 53 additions & 2 deletions src/sasctl/_services/model_management.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,13 @@ class ModelManagement(Service):
# TODO: set ds2MultiType
@classmethod
def publish_model(
cls, model, destination, name=None, force=False, reload_model_table=False
cls,
model,
destination,
model_version="latest",
name=None,
force=False,
reload_model_table=False,
):
"""

Expand All @@ -38,6 +44,8 @@ def publish_model(
The name or id of the model, or a dictionary representation of the model.
destination : str
Name of destination to publish the model to.
model_version_id : str or dict, optional
Provide the id, name, or dictionary representation of the version to publish. Defaults to 'latest'.
name : str, optional
Provide a custom name for the published model. Defaults to None.
force : bool, optional
Expand Down Expand Up @@ -68,6 +76,18 @@ def publish_model(

# TODO: Verify allowed formats by destination type.
# As of 19w04 MAS throws HTTP 500 if name is in invalid format.
if model_version != "latest":
if isinstance(model_version, dict) and "modelVersionName" in model_version:
model_version_name = model_version["modelVersionName"]
elif isinstance(model_version, str) and cls.is_uuid(model_version):
model_version_name = mr.get_model_or_version(model, model_version)[
"modelVersionName"
]
else:
model_version_name = model_version
else:
model_version_name = ""

model_name = name or "{}_{}".format(
model_obj["name"].replace(" ", ""), model_obj["id"]
).replace("-", "")
Expand All @@ -79,6 +99,7 @@ def publish_model(
{
"modelName": mp._publish_name(model_name),
"sourceUri": model_uri.get("uri"),
"modelVersionID": model_version_name,
"publishLevel": "model",
}
],
Expand All @@ -104,6 +125,7 @@ def create_performance_definition(
table_prefix,
project=None,
models=None,
modelVersions=None,
library_name="Public",
name=None,
description=None,
Expand Down Expand Up @@ -136,6 +158,9 @@ def create_performance_definition(
The name or id of the model(s), or a dictionary representation of the model(s). For
multiple models, input a list of model names, or a list of dictionaries. If no models are specified, all
models in the project specified will be used. Defaults to None.
modelVersions: str, list, optional
The name of the model version(s) for models used in the performance definition. If no model versions
are specified, all models will use the latest version. Defaults to None.
library_name : str
The library containing the input data, default is 'Public'.
name : str, optional
Expand Down Expand Up @@ -239,10 +264,36 @@ def create_performance_definition(
"property set." % project.name
)

if not modelVersions:
updated_models = [model.id for model in models]
else:
updated_models = []
if not isinstance(modelVersions, list):
modelVersions = [modelVersions]

if len(models) < len(modelVersions):
raise ValueError(
"There are too many versions for the amount of models specified."
)

modelVersions = modelVersions + [""] * (len(models) - len(modelVersions))
for model, modelVersionName in zip(models, modelVersions):
if (
isinstance(modelVersionName, dict)
and "modelVersionName" in modelVersionName
):
modelVersionName = modelVersionName["modelVersionName"]
elif (
isinstance(modelVersionName, dict)
and "modelVersionName" not in modelVersionName
):
raise ValueError("Model version is not recognized.")
updated_models.append(model.id + ":" + modelVersionName)

request = {
"projectId": project.id,
"name": name or project.name + " Performance",
"modelIds": [model.id for model in models],
"modelIds": [model for model in updated_models],
"championMonitored": monitor_champion,
"challengerMonitored": monitor_challenger,
"maxBins": max_bins,
Expand Down
3 changes: 2 additions & 1 deletion src/sasctl/_services/model_publish.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from .model_repository import ModelRepository
from .service import Service
from ..utils.decorators import deprecated


class ModelPublish(Service):
Expand Down Expand Up @@ -90,7 +91,7 @@ def delete_destination(cls, item):

return cls.delete("/destinations/{name}".format(name=item))

@classmethod
@deprecated("Use publish_model in model_management.py instead.", "1.11.5")
def publish_model(cls, model, destination, name=None, code=None, notes=None):
"""Publish a model to an existing publishing destination.

Expand Down
28 changes: 22 additions & 6 deletions src/sasctl/_services/score_definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def create_score_definition(
library_name: str, optional
The library within the CAS server the table exists in. Defaults to "Public".
model_version: str, optional
The user-chosen version of the model with the specified model_id. Defaults to "latest".
The user-chosen version of the model with the specified model version name. Defaults to latest version.

Returns
-------
Expand Down Expand Up @@ -116,7 +116,7 @@ def create_score_definition(
table = cls._cas_management.get_table(table_name, library_name, server_name)
if not table and not table_file:
raise HTTPError(
f"This table may not exist in CAS. Please include the `table_file` argument in the function call if it doesn't exist."
"This table may not exist in CAS. Please include the `table_file` argument in the function call if it doesn't exist."
)
elif not table and table_file:
cls._cas_management.upload_file(
Expand All @@ -125,16 +125,32 @@ def create_score_definition(
table = cls._cas_management.get_table(table_name, library_name, server_name)
if not table:
raise HTTPError(
f"The file failed to upload properly or another error occurred."
"The file failed to upload properly or another error occurred."
)
# Checks if the inputted table exists, and if not, uploads a file to create a new table

if model_version != "latest":

if isinstance(model_version, dict) and "modelVersionName" in model_version:
model_version = model_version["modelVersionName"]
elif isinstance(model_version, str) and cls.is_uuid(model_version):
model_version = cls._model_repository.get_model_or_version(
model_id, model_version
)["modelVersionName"]
else:
model_version = model_version

object_uri = f"/modelManagement/models/{model_id}/versions/@{model_version}"

else:
object_uri = f"/modelManagement/models/{model_id}"

save_score_def = {
"name": model_name, # used to be score_def_name
"description": description,
"objectDescriptor": {
"uri": f"/modelManagement/models/{model_id}",
"name": f"{model_name}({model_version})",
"uri": object_uri,
"name": f"{model_name} ({model_version})",
"type": f"{object_descriptor_type}",
},
"inputData": {
Expand All @@ -149,7 +165,7 @@ def create_score_definition(
"projectUri": f"/modelRepository/projects/{model_project_id}",
"projectVersionUri": f"/modelRepository/projects/{model_project_id}/projectVersions/{model_project_version_id}",
"publishDestination": "",
"versionedModel": f"{model_name}({model_version})",
"versionedModel": f"{model_name} ({model_version})",
},
"mappings": inputMapping,
}
Expand Down
Loading