Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Core] Fix A10 GPU on Azure #3707

Merged
merged 9 commits into from
Jul 5, 2024
Merged
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions sky/skylet/providers/azure/node_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,73 @@ def _create_node(self, node_config, tags, count):
create_or_update = get_azure_sdk_function(
client=self.resource_client.deployments, function_name="create_or_update"
)
poller = create_or_update(
resource_group_name=resource_group,
deployment_name=vm_name,
parameters=parameters,
)
poller.wait()

# pylint: disable=import-outside-toplevel
from sky.clouds.service_catalog import azure_catalog

instance_type = node_config["azure_arm_parameters"].get("vmSize", "")
accs = azure_catalog.get_accelerators_from_instance_type(instance_type)
if accs is None or "A10" not in accs:
return

# Configure driver extension for A10 GPUs
create_result = poller.result().as_dict()
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should merge the parameters for A10 to original parameters to avoid two create_or_update which can cause significant overhead? Please help profile it.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point! I benchmarked and it reduces the provision time from 15m15s to 14m36s.

output_resources = create_result.get("properties", {}).get(
"output_resources", []
)
vms_to_add_driver = []
for r in output_resources:
r_id = r.get("id", "")
if "Microsoft.Compute/virtualMachines" in r_id:
vms_to_add_driver.append(r_id.split("/")[-1])

for v in vms_to_add_driver:
logger.info(f"Begin to configure A10 driver extension for VM: {v}")
self._configure_a10_driver_extension(v)
logger.info(f"A10 driver extension configured for VM: {v}")

def _configure_a10_driver_extension(self, vm_name):
resource_group = self.provider_config["resource_group"]
parameters = {
"properties": {
"mode": DeploymentMode.incremental,
"template": {
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"vmName": {
"type": "string",
"metadata": {"description": "Name of the virtual machine"},
}
},
"resources": [
{
"type": "Microsoft.Compute/virtualMachines/extensions",
"apiVersion": "2015-06-15",
"location": "[resourceGroup().location]",
"name": "[concat(parameters('vmName'),'/NvidiaGpuDriverLinux')]",
"properties": {
"publisher": "Microsoft.HpcCompute",
"type": "NvidiaGpuDriverLinux",
"typeHandlerVersion": "1.9",
"autoUpgradeMinorVersion": True,
"settings": {},
},
}
],
},
"parameters": {"vmName": {"value": vm_name}},
}
}
create_or_update = get_azure_sdk_function(
client=self.resource_client.deployments, function_name="create_or_update"
)
create_or_update(
resource_group_name=resource_group,
deployment_name=vm_name,
Expand Down
Loading