Skip to content

Commit bbaa2e7

Browse files
dholman-modzydholman-modzy
committed
dev: adding in a progress commit for an initial implementation of the model-converter in the SDK with the demo notebook
1 parent d53492a commit bbaa2e7

File tree

7 files changed

+708
-0
lines changed

7 files changed

+708
-0
lines changed

‎model_converter_demo.ipynb

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# First we will convert an Explainable SageMaker Image Classification Model"
8+
]
9+
},
10+
{
11+
"cell_type": "code",
12+
"execution_count": 1,
13+
"metadata": {},
14+
"outputs": [],
15+
"source": [
16+
"blob_storage_provider = \"S3\"\n",
17+
"blob_storage_container = \"modzy-engineering-tests\"\n",
18+
"resources_key = \"ds/model-converter/sagemaker/image-classification/resources.tar.gz\"\n",
19+
"weights_key = \"ds/model-converter/sagemaker/image-classification/weights.tar.gz\""
20+
]
21+
},
22+
{
23+
"cell_type": "code",
24+
"execution_count": 2,
25+
"metadata": {
26+
"pycharm": {
27+
"name": "#%%\n"
28+
}
29+
},
30+
"outputs": [],
31+
"source": [
32+
"from modzy.converter.model_converter import ModelConverter\n",
33+
"from modzy.client import ApiClient\n",
34+
"import os\n",
35+
"\n",
36+
"# To get started, store your Modzy API key as an environment variable `MODZY_API_KEY`.\n",
37+
"# Then, create a Modzy API client to interact with the integration envrionment\n",
38+
"modzy_api_key = os.getenv(\"MODZY_QA_API_KEY\")\n",
39+
"modzy_instance_base_url = \"https://integration.modzy.engineering/api\"\n",
40+
"modzy_api_client = ApiClient(api_key=modzy_api_key, base_url=modzy_instance_base_url)\n",
41+
"\n",
42+
"# Instantiate a Model Converter client with access to the Modzy integration environment\n",
43+
"model_converter = ModelConverter(modzy_api_client)"
44+
]
45+
},
46+
{
47+
"cell_type": "code",
48+
"execution_count": 3,
49+
"metadata": {
50+
"pycharm": {
51+
"name": "#%%\n"
52+
}
53+
},
54+
"outputs": [
55+
{
56+
"name": "stdout",
57+
"output_type": "stream",
58+
"text": [
59+
"The model details page for your new model can be found here: https://integration.modzy.engineering/models/106d50ca0e-modzy-image-classification/0.0.1\n"
60+
]
61+
}
62+
],
63+
"source": [
64+
"# Now, provide the Model converter with information about your stored model assets and the credentials required\n",
65+
"# to access them. The Model converter will do the rest of the work.\n",
66+
"\n",
67+
"source_platform = \"sagemaker\"\n",
68+
"model_type = \"image-classification\"\n",
69+
"\n",
70+
"_, converter_output = model_converter.run(\n",
71+
" sp_access_key_id=os.getenv(\"SP_ACCESS_KEY_ID\"),\n",
72+
" sp_secret_access_key=os.getenv(\"SP_SECRET_ACCESS_KEY\"),\n",
73+
" blobstore_provider=blob_storage_provider,\n",
74+
" blobstore_container=blob_storage_container,\n",
75+
" weights_path=weights_key,\n",
76+
" resources_path=resources_key,\n",
77+
" platform=source_platform,\n",
78+
" model_type=model_type,\n",
79+
")\n",
80+
"\n",
81+
"print(f\"The model details page for your new model can be found here: {converter_output['modelURL']}\")\n",
82+
"new_model_id = converter_output[\"modelId\"]\n",
83+
"new_model_version = converter_output[\"modelVersion\"]"
84+
]
85+
},
86+
{
87+
"cell_type": "code",
88+
"execution_count": 4,
89+
"metadata": {
90+
"pycharm": {
91+
"name": "#%%\n"
92+
}
93+
},
94+
"outputs": [],
95+
"source": [
96+
"# Delegate a single processing to serve your new model\n",
97+
"modzy_api_client.models.update_processing_engines(new_model_id, new_model_version, min_engines=1, max_engines=1)"
98+
]
99+
},
100+
{
101+
"cell_type": "code",
102+
"execution_count": null,
103+
"metadata": {
104+
"pycharm": {
105+
"name": "#%%\n"
106+
}
107+
},
108+
"outputs": [],
109+
"source": [
110+
"# Send an inference job to run against your new model with explainability!\n",
111+
"input_source = {\n",
112+
" \"0001\": {\n",
113+
" f\"image\": {\n",
114+
" \"bucket\": blob_storage_container,\n",
115+
" \"key\": f\"/ds/model-converter/{source_platform}/{model_type}/test_input\"\n",
116+
" }\n",
117+
" }\n",
118+
"}\n",
119+
"\n",
120+
"print(f\"Sending job to model {new_model_id} {new_model_version}\")\n",
121+
"job = modzy_api_client.jobs.submit_aws_s3(\n",
122+
" new_model_id, new_model_version, input_source,\n",
123+
" os.getenv(\"SP_ACCESS_KEY_ID\"), os.getenv(\"SP_SECRET_ACCESS_KEY\"),\n",
124+
" region=\"us-east-1\", explain=True\n",
125+
")\n",
126+
"\n",
127+
"modzy_api_client.jobs.block_until_complete(job, timeout=None)\n",
128+
"print(\"Job Completed!\")"
129+
]
130+
},
131+
{
132+
"cell_type": "markdown",
133+
"metadata": {},
134+
"source": [
135+
"# Next, we will convert an MLflow model"
136+
]
137+
},
138+
{
139+
"cell_type": "code",
140+
"execution_count": null,
141+
"metadata": {
142+
"pycharm": {
143+
"name": "#%%\n"
144+
}
145+
},
146+
"outputs": [],
147+
"source": [
148+
"from modzy.converter.mlflow import upload_mlflow_model\n",
149+
"\n",
150+
"# Raw output MLFlow Model Gets uploaded to the user's S3 Bucket\n",
151+
"# upload_mlflow_model()\n",
152+
"\n",
153+
"# Now we repeat the process with an MLFlow model\n",
154+
"source_platform = \"mlflow\"\n",
155+
"model_type = \"tabular\"\n",
156+
"resources_key = \"ds/model-converter/mlflow/tabular/resources.tar.gz\"\n",
157+
"weights_key = \"ds/model-converter/mlflow/tabular/model.tar.gz\"\n",
158+
"\n",
159+
"\n",
160+
"_, converter_output = model_converter.run(\n",
161+
" sp_access_key_id=os.getenv(\"SP_ACCESS_KEY_ID\"),\n",
162+
" sp_secret_access_key=os.getenv(\"SP_SECRET_ACCESS_KEY\"),\n",
163+
" blobstore_provider=blob_storage_provider,\n",
164+
" blobstore_container=blob_storage_container,\n",
165+
" weights_path=weights_key,\n",
166+
" resources_path=resources_key,\n",
167+
" platform=source_platform,\n",
168+
" model_type=model_type,\n",
169+
")\n",
170+
"\n",
171+
"print(f\"The model details page for your new model can be found here: {converter_output['modelURL']}\")\n",
172+
"new_model_id = converter_output[\"modelId\"]\n",
173+
"new_model_version = converter_output[\"modelVersion\"]"
174+
]
175+
},
176+
{
177+
"cell_type": "code",
178+
"execution_count": null,
179+
"metadata": {
180+
"pycharm": {
181+
"name": "#%%\n"
182+
}
183+
},
184+
"outputs": [],
185+
"source": [
186+
"# Delegate a single processing to serve your new model\n",
187+
"modzy_api_client.models.update_processing_engines(\"106d50ca0e\", \"0.0.1\", min_engines=0, max_engines=1)"
188+
]
189+
},
190+
{
191+
"cell_type": "code",
192+
"execution_count": null,
193+
"metadata": {},
194+
"outputs": [],
195+
"source": []
196+
}
197+
],
198+
"metadata": {
199+
"kernelspec": {
200+
"display_name": "Python 3",
201+
"language": "python",
202+
"name": "python3"
203+
},
204+
"language_info": {
205+
"codemirror_mode": {
206+
"name": "ipython",
207+
"version": 3
208+
},
209+
"file_extension": ".py",
210+
"mimetype": "text/x-python",
211+
"name": "python",
212+
"nbconvert_exporter": "python",
213+
"pygments_lexer": "ipython3",
214+
"version": "3.9.1"
215+
}
216+
},
217+
"nbformat": 4,
218+
"nbformat_minor": 1
219+
}

‎modzy/converter/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy