<jupyter_start><jupyter_text>Get workspace and authenticate<jupyter_code>import json
import azureml.core
from azureml.core import Workspace, Datastore, Dataset

print(azureml.core.VERSION)
ws = Workspace.from_config()
print(ws.name, ws.subscription_id, ws.resource_group)
ws<jupyter_output><empty_output><jupyter_text>query Key Vault<jupyter_code>kv = c
print(len(kv.get_secret('adfserviceprincipal')))<jupyter_output>32<jupyter_text>Create a data store<jupyter_code>ws.datastores
print(type(ws.datastores.items()))

for i,v in ws.datastores.items(): 
    print(i)
    print(type(v))
    print(v.name)
    print(v.container_name)
    print(v.account_name)
    print(v.protocol)
    print(v.endpoint)

    
datastore_name ='godzilla'
account_name = 'godzillasinastorage'
filesystem_name = 'batchml'
Datastore.get_default(workspace=ws)

serviceprincipal = json.loads( kv.get_secret('magamlworkspace'))
print(serviceprincipal.keys())
print(serviceprincipal['clientid'])

storageaccesskey = json.loads(kv.get_secret('godzillakey'))
print(storageaccesskey.keys())
# Datastore.register_azure_data_lake_gen2(
#     workspace=ws,
#     datastore_name=datastore_name,
#     account_name=account_name, # ADLS Gen2 account name
#     filesystem=filesystem_name, # ADLS Gen2 filesystem
#     tenant_id=serviceprincipal['tenant'], # tenant id of service principal
#     client_id=serviceprincipal['clientid'], # client id of service principal
#     client_secret=serviceprincipal['secret'] # the secret of service principal
#     )
Datastore.register_azure_blob_container(
    workspace=ws, 
    datastore_name=datastore_name, 
    container_name=storageaccesskey['container'], 
    account_name=storageaccesskey['accountname'], 
    account_key=storageaccesskey['key'],
    overwrite=True
    )
godzilla_datastore = Datastore.get(
    workspace=ws, 
    datastore_name=datastore_name
)<jupyter_output><empty_output><jupyter_text>Create and Register Env<jupyter_code>from azureml.core import Environment
from azureml.core.conda_dependencies import CondaDependencies
from azureml.core.runconfig import DEFAULT_GPU_IMAGE

#print([env for env in ws.environments])

print([cd for cd in ws.environments['MAG-ParallelRunEnv'].python.conda_dependencies.conda_packages])
print([cd for cd in ws.environments['MAG-ParallelRunEnv'].python.conda_dependencies.pip_packages])


#"tensorflow-gpu==1.15.2",
cd = CondaDependencies.create(pip_packages=["pandas",
                                            "azureml-core", "azureml-dataprep[fuse]"])
CondaDependencies
env = Environment(name="MAG-ParallelRunEnv")
env.python.conda_dependencies=cd
env.docker.base_image = DEFAULT_GPU_IMAGE
env.register(workspace=ws)<jupyter_output><empty_output><jupyter_text>create and register images data setFileDataset can be used as input of an experiment run. It can also be registered to workspace with a specified name and be retrieved by that name later.FileDataset can be subsetted by invoking different subsetting methods available on this class. The result of subsetting is always a new FileDataset.The actual data loading happens when FileDataset is asked to deliver the data into another storage mechanism (e.g. files downloaded or mounted to local path).<jupyter_code>dataset_path = (godzilla_datastore, 'images')

dataset_name = 'anpr_images'
anpr_images = ws.datasets.get(dataset_name)
if anpr_images is None: 
    print(f'dataset:{dataset_name} does not exit, create and register')
    anpr_images = Dataset.File.from_files(dataset_path)
    anpr_images = anpr_images.register(workspace=ws, name=dataset_name)
    print(anpr_images)
anpr_images.to_path()
# with anpr_images.mount() as mount_context: 
#     os.listdir(mount_context.mount_point)
# # You can also use the start and stop methods
# mount_context = anpr_images.mount()
# print(mount_context.mount_point)
# mount_context.start()  # this will mount the file streams
# mount_context.stop()  # this will unmount the file streams<jupyter_output><empty_output><jupyter_text>abstract data set operationsexploring what we get with file datasets<jupyter_code>print(type(anpr_images))
named_input = anpr_images.as_named_input('foo')
print(named_input.mode, named_input.name)
ws.datasets
ds_anpr = ws.datasets['anpr_images']
type(ds_anpr)
ds_anpr.to_path()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup