Does anyone have experience with the Python SDK (v: v2.0.0rc6) and cloning/duplicating running VM's into another resource group?
getting the OS disk to start. Will need the data disk as well
managed_disk = self.compute_client.disks.get(resource_group_name=source_rg_name, disk_name=vm.storage_profile.os_disk.name)
make a snapshot of the os disk.
self.compute_client.snapshots.create_or_update(
self.config.resource_group_name,
'SNAPSHOT-' + virtual_machine,
{
'location': managed_disk.location,
'creation_data': {
'create_option': 'Copy',
'source_uri': managed_disk.id
}
}
)
create the VM. Throws exception below.
result = self.compute_client.virtual_machines.create_or_update(
self.config.cybric_resource_group_name,
virtual_machine,
azure.mgmt.compute.models.VirtualMachine(
location=vm.location,
os_profile=vm.os_profile,
hardware_profile=vm.hardware_profile,
network_profile=azure.mgmt.compute.models.NetworkProfile(
network_interfaces=[
azure.mgmt.compute.models.NetworkInterfaceReference(
id=nic_obj['id'],
primary=True
),
],
),
storage_profile=azure.mgmt.compute.models.StorageProfile(
os_disk=azure.mgmt.compute.models.OSDisk(
caching=azure.mgmt.compute.models.CachingTypes.none,
create_option=azure.mgmt.compute.models.DiskCreateOptionTypes.attach,
name=dup_virtual_machine,
managed_disk=azure.mgmt.compute.models.ManagedDiskParameters(
id=managed_disk.id
),
),
image_reference = azure.mgmt.compute.models.ImageReference(
publisher=vm.storage_profile.image_reference.publisher,
offer=vm.storage_profile.image_reference.offer,
sku=vm.storage_profile.image_reference.sku,
version=vm.storage_profile.image_reference.version,
),
),
),
)
Exception:
Failed to create virtual machines: Azure Error: InvalidParameter
Message: Cannot attach an existing OS disk if the VM is created from a
platform or user image. Target: osDisk
The error log reason is that create_option is wrong, you should use FromImage not attach. You could check Azure Python SDK in this link.
Related
I am trying to write a code which will read values from excel file and will create VMs in Google Cloud. I am facing problem at two locations, while creating tags if I use 'items': [tag] or while creating service account scope it starts giving me error.
import os, json
import googleapiclient.discovery
from google.oauth2 import service_account
import csv
credentials = service_account.Credentials.from_service_account_file('G:/python/json/mykids-280210.json')
compute = googleapiclient.discovery.build('compute', 'v1', credentials=credentials)
def create_instance(compute, vm_name, image_project, image_family, machinetype, startupscript, zone, network,
subnet, project, scope, tag):
# Get the latest Debian Jessie image.
image_response = compute.images().getFromFamily(
project=image_project, family=image_family).execute()
source_disk_image = image_response['selfLink']
# Configure the machine
machine_type = "zones/" + zone + "/machineTypes/" + machinetype
startup_script = startupscript
config = {
'name': vm_name,
'machineType': machine_type,
'description': 'This VM was created with python code',
'tags': {
'items': ['external', 'home', 'local'] #'items': [tag] <~~~~~~~~~~~
},
'deletionProtection': False,
'labels': {'env': 'dev', 'server': 'mytower', 'purpose': 'personal'},
# Specify the boot disk and the image to use as a source.
'disks': [
{
'boot': True,
'autoDelete': True,
'initializeParams': {
'sourceImage': source_disk_image,
}
}
],
# Specify a network interface with NAT to access the public
# internet.
'networkInterfaces': [{
'network': 'global/networks/' + network,
'subnetwork': 'regions/us-central1/subnetworks/' + subnet,
'accessConfigs': [
{'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT'}
]
}],
# Allow the instance to access cloud storage and logging.
'serviceAccounts': [{
'email': 'default',
'scopes': [
#'https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write'
#scope # scope <~~~~~~~~~~~~~~~~~~~~
]
}],
'scheduling': {
"preemptible": True
},
# Metadata is readable from the instance and allows you to
# pass configuration from deployment scripts to instances.
'metadata': {
'items': [{
# Startup script is automatically executed by the
# instance upon startup.
'key': 'startup-script',
'value': startup_script
}]
}
}
return compute.instances().insert(
project=project,
zone=zone,
body=config).execute()
# [END create_instance]
with open('vms.csv', newline='') as csvfile:
data = csv.DictReader(csvfile)
for row in data:
vm_name = row['vm_name']
image_project = row['image_project']
image_family = row['image_family']
machinetype = row['machinetype']
startupscript = row['startupscript']
zone = row['zone']
network = row['network']
subnet = row['subnet']
project = row['project']
scope = row['scopes']
tag = row['tags']
print(create_instance(compute, vm_name, image_project, image_family, machinetype, startupscript, zone, network,
subnet, project, scope, tag))
csvfile.close()
error when use scope variable
G:\python\pythonProject\venv\Scripts\python.exe G:/python/pythonProject/read-excel-gcp/vm/create_vm.py
Traceback (most recent call last):
File "G:\python\pythonProject\read-excel-gcp\vm\create_vm.py", line 100, in <module>
print(create_instance(compute, vm_name, image_project, image_family, machinetype, startupscript, zone, network,
File "G:\python\pythonProject\read-excel-gcp\vm\create_vm.py", line 79, in create_instance
return compute.instances().insert(
File "G:\python\pythonProject\venv\lib\site-packages\googleapiclient\_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "G:\python\pythonProject\venv\lib\site-packages\googleapiclient\http.py", line 915, in execute
raise HttpError(resp, content, uri=self.uri)
googleapiclient.errors.HttpError: <HttpError 400 when requesting https://compute.googleapis.com/compute/v1/projects/mykids-280210/zones/us-central1-a/instances?alt=json returned "One or more of the service account scopes are invalid: 'https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write''". Details: "One or more of the service account scopes are invalid: 'https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write''">
Process finished with exit code 1
I get a similar error when I use tag variable.
I have # the value the way I am passing in the above code.
Below is my csv file details
vm_name,image_project,image_family,machinetype,startupscript,zone,network,subnet,project,scopes,tags
python-vm1,debian-cloud,debian-9,e2-micro,G:/python/json/startup-script.sh,us-central1-a,myvpc,subnet-a,mykids-280210,"https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write'","external', 'home', 'local'"
python-vm2,debian-cloud,debian-9,e2-micro,G:/python/json/startup-script.sh,us-central1-a,myvpc,subnet-a,mykids-280210,"https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write'","external', 'home', 'local'"
I am not sure that when the value are passed directly it works, but when passing the value through variable, it fails.
I have marked the problem area with <~~~~~~~~~~~~
Please suggest if anyone understands the issue.
#d.s can you try changing your scope format to something like this:
'serviceAccounts': [
{
'email': 'default'
'scopes':[
'https://www.googleapis.com/auth/compute',
'https://www.googleapis.com/auth/servicecontrol',
'https://www.googleapis.com/auth/service.management.readonly',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/monitoring.write',
'https://www.googleapis.com/auth/trace.append',
'https://www.googleapis.com/auth/devstorage.read_write']}]
The listed scopes are the default scopes that you will need for an instance. I think the problem you are facing is you where trying to only list two scopes which are not enough to allow you to deploy your instance.
I've followed the documentation pretty well as outlined here.
I've setup my azure machine learning environment the following way:
from azureml.core import Workspace
# Connect to the workspace
ws = Workspace.from_config()
from azureml.core import Environment
from azureml.core import ContainerRegistry
myenv = Environment(name = "myenv")
myenv.inferencing_stack_version = "latest" # This will install the inference specific apt packages.
# Docker
myenv.docker.enabled = True
myenv.docker.base_image_registry.address = "myazureregistry.azurecr.io"
myenv.docker.base_image_registry.username = "myusername"
myenv.docker.base_image_registry.password = "mypassword"
myenv.docker.base_image = "4fb3..."
myenv.docker.arguments = None
# Environment variable (I need python to look at folders
myenv.environment_variables = {"PYTHONPATH":"/root"}
# python
myenv.python.user_managed_dependencies = True
myenv.python.interpreter_path = "/opt/miniconda/envs/myenv/bin/python"
from azureml.core.conda_dependencies import CondaDependencies
conda_dep = CondaDependencies()
conda_dep.add_pip_package("azureml-defaults")
myenv.python.conda_dependencies=conda_dep
myenv.register(workspace=ws) # works!
I have a score.py file configured for inference (not relevant to the problem I'm having)...
I then setup inference configuration
from azureml.core.model import InferenceConfig
inference_config = InferenceConfig(entry_script="score.py", environment=myenv)
I setup my compute cluster:
from azureml.core.compute import ComputeTarget, AksCompute
from azureml.exceptions import ComputeTargetException
# Choose a name for your cluster
aks_name = "theclustername"
# Check to see if the cluster already exists
try:
aks_target = ComputeTarget(workspace=ws, name=aks_name)
print('Found existing compute target')
except ComputeTargetException:
print('Creating a new compute target...')
prov_config = AksCompute.provisioning_configuration(vm_size="Standard_NC6_Promo")
aks_target = ComputeTarget.create(workspace=ws, name=aks_name, provisioning_configuration=prov_config)
aks_target.wait_for_completion(show_output=True)
from azureml.core.webservice import AksWebservice
# Example
gpu_aks_config = AksWebservice.deploy_configuration(autoscale_enabled=False,
num_replicas=3,
cpu_cores=4,
memory_gb=10)
Everything succeeds; then I try and deploy the model for inference:
from azureml.core.model import Model
model = Model(ws, name="thenameofmymodel")
# Name of the web service that is deployed
aks_service_name = 'tryingtodeply'
# Deploy the model
aks_service = Model.deploy(ws,
aks_service_name,
models=[model],
inference_config=inference_config,
deployment_config=gpu_aks_config,
deployment_target=aks_target,
overwrite=True)
aks_service.wait_for_deployment(show_output=True)
print(aks_service.state)
And it fails saying that it can't find the environment. More specifically, my environment version is version 11, but it keeps trying to find an environment with a version number that is 1 higher (i.e., version 12) than the current environment:
FailedERROR - Service deployment polling reached non-successful terminal state, current service state: Failed
Operation ID: 0f03a025-3407-4dc1-9922-a53cc27267d4
More information can be found here:
Error:
{
"code": "BadRequest",
"statusCode": 400,
"message": "The request is invalid",
"details": [
{
"code": "EnvironmentDetailsFetchFailedUserError",
"message": "Failed to fetch details for Environment with Name: myenv Version: 12."
}
]
}
I have tried to manually edit the environment JSON to match the version that azureml is trying to fetch, but nothing works. Can anyone see anything wrong with this code?
Update
Changing the name of the environment (e.g., my_inference_env) and passing it to InferenceConfig seems to be on the right track. However, the error now changes to the following
Running..........
Failed
ERROR - Service deployment polling reached non-successful terminal state, current service state: Failed
Operation ID: f0dfc13b-6fb6-494b-91a7-de42b9384692
More information can be found here: https://some_long_http_address_that_leads_to_nothing
Error:
{
"code": "DeploymentFailed",
"statusCode": 404,
"message": "Deployment not found"
}
Solution
The answer from Anders below is indeed correct regarding the use of azure ML environments. However, the last error I was getting was because I was setting the container image using the digest value (a sha) and NOT the image name and tag (e.g., imagename:tag). Note the line of code in the first block:
myenv.docker.base_image = "4fb3..."
I reference the digest value, but it should be changed to
myenv.docker.base_image = "imagename:tag"
Once I made that change, the deployment succeeded! :)
One concept that took me a while to get was the bifurcation of registering and using an Azure ML Environment. If you have already registered your env, myenv, and none of the details of the your environment have changed, there is no need re-register it with myenv.register(). You can simply get the already register env using Environment.get() like so:
myenv = Environment.get(ws, name='myenv', version=11)
My recommendation would be to name your environment something new: like "model_scoring_env". Register it once, then pass it to the InferenceConfig.
I have a Windows Machine that I want to add VM extension using the azure python SDK , I send the following request
{'location': 'westus',
'tags': None,
'publisher': 'Microsoft.Compute',
'virtual_machine_extension_type': 'CustomScriptExtension',
'type_handler_version': '1.4',
'settings': '{
"file_uris": ["https://mysite.azurescripts.net/ps_enable_winrm_http.ps1"],
"command_to_execute": "powershell -ExecutionPolicy Unrestricted -file ps_enable_winrm_http.ps1"}'
}
but what happens is that it gives the following exception
configure virtual_machine '946b4246-a604-4b01-9e6a-09ed64a93bdb' failed with this error :
VM has reported a failure when processing extension '13da0dc5-09c0-4e56-a35d-fdbc42432e11'.
Error message: "Invalid handler configuration. Exiting.
Error Message: Expecting state 'Element'.. Encountered 'Text' with name '', namespace ''. "
More information on troubleshooting is available at https://aka.ms/VMExtensionCSEWindowsTroubleshoot
adding a simple code snippet that I use
vm_extension_name = "{0}".format(uuid4())
vm_extension_params = {
'location': location_val,
'tags': tags_val,
'publisher': 'Microsoft.Compute',
'virtual_machine_extension_type': 'CustomScriptExtension',
'type_handler_version': type_handler_version,
'auto_upgrade_minor_version': True,
'settings': json.dumps({
'fileUris': file_uris,
'commandToExecute': command_to_execute
})
}
logger.info("sending {0}".format(vm_extension_params))
any ideas , should I send something differently or am I missing something from the above request that cause the issue
thanks for the help in advance
Regards,
When we use python sdk to install custom script extension, we should create Object VirtualMachineExtension. Its parameter settings should be Object. But you define it as str. Please update it with removing ''. For more details, please refer to the document
For example
from azure.mgmt.compute import ComputeManagementClient
from azure.common.credentials import ServicePrincipalCredentials
AZURE_TENANT_ID= ''
AZURE_CLIENT_ID=''
AZURE_CLIENT_SECRET=''
AZURE_SUBSCRIPTION_ID=''
credentials = ServicePrincipalCredentials(client_id=AZURE_CLIENT_ID,secret=AZURE_CLIENT_SECRET,tenant=AZURE_TENANT_ID)
compute_client = ComputeManagementClient(credentials, AZURE_SUBSCRIPTION_ID)
resource_group_name='stan'
vm_name='win2016'
params_create = {
'location':'CentralUS',
'tags': None,
'publisher': 'Microsoft.Compute',
'virtual_machine_extension_type': 'CustomScriptExtension',
'type_handler_version': '1.4',
'settings':
{
'fileUris': ['https://***/test/test.ps1'],
'commandToExecute': 'powershell -ExecutionPolicy Unrestricted -File test.ps1'
}
}
ext_poller = compute_client.virtual_machine_extensions.create_or_update(
resource_group_name,
vm_name,
'test',
params_create,
)
ext = ext_poller.result()
print(ext)
I want to monitor and get information regarding the different instances in an Azure Virtual Machine Scale Set (VMSS).
I used the command (Python):
vmss = compute_client.virtual_machine_scale_sets.list(resource_group, scale_set_name)
But I am not able to get the result I am expecting.
Any suggestions what to do?
You can use the following code to get the ip and powerstate.
compute_client = ComputeManagementClient(credentials, subscription_id)
vmss = compute_client.virtual_machine_scale_set_vms.list(resource_group_name="", vmss="")
for item in vmss:
print("name: ", item.name)
ni_reference = item.network_profile.network_interfaces[0].id
resource_client = ResourceManagementClient(credentials, subscription_id)
nic = resource_client.resources.get_by_id(
ni_reference,
api_version='2017-12-01')
ip_reference = nic.properties['ipConfigurations'][0]['properties']
print("ip info: ", ip_reference)
instance_view = compute_client.virtual_machine_scale_set_vms.get_instance_view(resource_group_name="", vmss="", instance_id=item.instance_id)
print(instance_view.statuses[1].code)
result:
name: yangtestvmss_1
ip info: {'provisioningState': 'Succeeded', 'privateIPAddress': '10.0.0.5', 'privateIPAllocationMethod': 'Dynamic', 'subnet': {'id': '/subscriptions/e5b0fcfa-e859-43f3-8d84-5e5fe29f4c68/resourceGroups/yangtestvmss/providers/Microsoft.Network/virtualNetworks/yangtestvmssVnet/subnets/default'}, 'primary': True, 'privateIPAddressVersion': 'IPv4', 'isInUseWithService': False}
PowerState/running
name: yangtestvmss_3
ip info: {'provisioningState': 'Succeeded', 'privateIPAddress': '10.0.0.7', 'privateIPAllocationMethod': 'Dynamic', 'subnet': {'id': '/subscriptions/e5b0fcfa-e859-43f3-8d84-5e5fe29f4c68/resourceGroups/yangtestvmss/providers/Microsoft.Network/virtualNetworks/yangtestvmssVnet/subnets/default'}, 'primary': True, 'privateIPAddressVersion': 'IPv4', 'isInUseWithService': False}
PowerState/running
If you want to get the VMs information, please use the following code.
subscription_id = 'subscription Id'
credentials = ServicePrincipalCredentials(client_id=CLIENT, secret=KEY, tenant=TENANT_ID)
client = ComputeManagementClient(credentials, subscription_id)
vmss = client.virtual_machine_scale_set_vms.list("resourcegroup Name","VMSS name")
for item in vmss:
print("id:",item.id)
print("name",item.name)
Test Result:
There is a cool tool that a guy from Microsoft has been build for monitoring VMSS
see this link VMSS Dashboard
The mentioned tool helps you to see the status of VMs in the scale set: you can see the update domain and fault domain grouping of VMs. It lets you start or deallocate a VM. The code is for more than two years ago.
I am new to docker as well as azure batch. The problem i am having currently is i have 2 dotnet console applications one of them runs locally (which creates the pool, job and task on azure batch programmatically) and for second one i have created a docker image and pushed to azure container registry. Now the things is when i create the cloudtTask from locally running application as monetione below
TaskContainerSettings cmdContainerSettings = new TaskContainerSettings(
imageName: "myrepository.azurecr.io/pipeline:latest",
containerRunOptions: "--rm"
);
CloudTask containerTask = new CloudTask(
id: "task1",
commandline: cmdLine);
containerTask.ContainerSettings = cmdContainerSettings;
Console.WriteLine("Task created");
await batchClient.JobOperations.AddTaskAsync(newJobId, containerTask);
Console.WriteLine("-----------------------");
and add it to the BatchClient, the expcetion i get in azure batch (Azure portal) is this:
System.UnauthorizedAccessException: Access to the path '/home/_azbatch/.dotnet' is denied. ---> System.IO.IOException: Permission denied
--- End of inner exception stack trace ---
What can be the problem? Thank you.
As the comment ended up being the answer, I'm posting it here for clarity for future viewers:
The task needs to be run with elevated rights.
eg.
containerTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
See the docs for more info
i am still not able to pull image from docker, i am using nodejs .. following are configs for creating task
const taskConfig = {
"id": "task-new-2",
"commandLine": "bash -c 'node index.js'",
"containerSettings": {
"imageName": "xxx.xx.io/xx-test:latest",
"containerRunOptions": "--rm",
"username": "xxx",
"password": "tfDlZ",
"registryServer": "xxx.xx.io",
// "workingDirectory": "AZ_BATCH_NODE_ROOT_DIR"
},
"userIdentity": {
"autoUser": {
"scope": "pool",
"elevationLevel": "admin"
}
}
}