Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added more configurable parameters to the script. #67

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Added more configurable parameters to the script. #67

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 105 additions & 80 deletions clone_dataset/clone_dataset_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,17 @@
# do you want to clone only the first layer or all of the layers?
clone_first_layer_only = False

# Application for which you want to clone from and to. The source application
# does not apply to datasets (it is cloned regardless) but only to layers,
# widgets, metadata and vocabularies
source_application = 'rw'
cloned_application = 'rw'

# do you want to clone only the default editable widget or all the widgets?
clone_default_widget_only = True
clone_default_widget_only = False

# Do you want to do a dry run and just print info on what will be done?
dry_run = False


API_TOKEN = os.getenv('RW_API_KEY')
Expand Down Expand Up @@ -68,7 +76,7 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase
clone_dataset_attr[k] = dataset_params.get(k, '')
payload = {
'dataset': {
'application': ['rw'],
'application': [cloned_application],
'connectorType': clone_dataset_attr['connectorType'],
'connectorUrl': clone_dataset_attr['connectorUrl'],
'tableName': clone_dataset_attr['tableName'],
Expand All @@ -95,35 +103,42 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase
print(f'Creating clone dataset')
url = f'{clone_server}/dataset'
headers = {'Authorization': f'Bearer {token}', 'Content-Type': 'application/json', 'Cache-Control': 'no-cache'}
r = requests.post(url, data=json.dumps(payload), headers=headers)
if r.status_code == 200:
clone_dataset_id = r.json()['data']['id']
clone_dataset = LMIPy.Dataset(id_hash=clone_dataset_id, server=clone_server)
else:
print(r.status_code)
print(f'{clone_server}/v1/dataset/{clone_dataset_id}')
if not dry_run:
r = requests.post(url, data=json.dumps(payload), headers=headers)
if r.status_code == 200:
clone_dataset_id = r.json()['data']['id']
clone_dataset = LMIPy.Dataset(id_hash=clone_dataset_id, server=clone_server)
else:
print(r.status_code)
print(f'{clone_server}/v1/dataset/{clone_dataset_id}')
if clone_children:
layers = self.layers
if len(layers) > 0:
if clone_first_layer_only:
l = layers[0]
if l.attributes['application'] == ['rw']:
if source_application in l.attributes['application']:
try:
layer_name = l.attributes['name']
print('Cloning layer: {}'.format(layer_name))
l.clone(token=token, env='production', layer_params={'name': layer_name},
target_dataset_id=clone_dataset_id)
time.sleep(2)
if not dry_run:
l.clone(token=token, env='production',
layer_params={
'name': layer_name,
'application': [cloned_application]
},
target_dataset_id=clone_dataset_id)
time.sleep(2)
except:
raise ValueError(f'Layer cloning failed for {l.id}')
else:
for l in layers:
if l.attributes['application']==['rw']:
if source_application in l.attributes['application']:
try:
layer_name = l.attributes['name']
print('Cloning layer: {}'.format(layer_name))
l.clone(token=token, env='production', layer_params={'name': layer_name}, target_dataset_id=clone_dataset_id)
time.sleep(2)
if not dry_run:
l.clone(token=token, env='production', layer_params={'name': layer_name}, target_dataset_id=clone_dataset_id)
time.sleep(2)
except:
raise ValueError(f'Layer cloning failed for {l.id}')
else:
Expand All @@ -145,36 +160,35 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase
widget_config = widget['widgetConfig'],
app = widget['application']
ds_id = clone_dataset_id
if app == ['rw']:
if source_application in app :
if name and widget_config and app:
widget_payload = {
"name": widget['name'],
"description": widget.get('description', None),
"env": widget['env'],
"widgetConfig": widget['widgetConfig'],
"application": ['rw']
"application": [cloned_application]
}
try:
url = f'{self.server}/v1/dataset/{ds_id}/widget'
print(url)
headers = {'Authorization': 'Bearer ' + token,
'Content-Type': 'application/json'}
r = requests.post(url, data=json.dumps(widget_payload), headers=headers)
print(r.json())
except:
raise ValueError(f'Widget creation failed.')
if r.status_code == 200:
print(f'Widget created.')
# self.attributes = self.get_dataset()
else:
print(f'Failed with error code {r.status_code}')
print(f"Cloning widget {widget['name']}")
if not dry_run:
try:
url = f'{self.server}/v1/dataset/{ds_id}/widget'
headers = {'Authorization': 'Bearer ' + token,
'Content-Type': 'application/json'}
r = requests.post(url, data=json.dumps(widget_payload), headers=headers)
print(r.json())
except:
raise ValueError(f'Widget creation failed.')
if r.status_code == 200:
print(f'Widget created.')
# self.attributes = self.get_dataset()
else:
print(f'Failed with error code {r.status_code}')
else:
raise ValueError(
f'Widget creation requires name string, application list and a widgetConfig object.')

# clone_dataset.add_widget(token=token, widget_params=widget_payload)
else:
print("Non-rw app. Not cloning.")
except:
raise ValueError(f'Widget cloning failed for {widget.id}')
else:
Expand All @@ -184,79 +198,90 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase
name = widget['name'],
widget_config = widget['widgetConfig'],
app = widget['application']
ds_id = clone_dataset_id
if app==['rw']:
print('Widget application - ', widget['application'])
if source_application in app:
if name and widget_config and app:
print(f"Cloning widget {widget['name']}")
widget_payload = {
"name": widget['name'],
"description": widget.get('description', None),
"env": widget['env'],
"widgetConfig": widget['widgetConfig'],
"application": ['rw']
"application": [cloned_application]
}
try:
url = f'{self.server}/v1/dataset/{ds_id}/widget'
print(url)
headers = {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}
r = requests.post(url, data=json.dumps(widget_payload), headers=headers)
print(r.json())
except:
raise ValueError(f'Widget creation failed.')
if r.status_code == 200:
print(f'Widget created.')
#self.attributes = self.get_dataset()
else:
print(f'Failed with error code {r.status_code}')
if not dry_run:
ds_id = clone_dataset_id
try:
url = f'{self.server}/v1/dataset/{ds_id}/widget'
headers = {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}
r = requests.post(url, data=json.dumps(widget_payload), headers=headers)
print(r.json())
except:
raise ValueError(f'Widget creation failed.')
if r.status_code == 200:
print(f'Widget created.')
#self.attributes = self.get_dataset()
else:
print(f'Failed with error code {r.status_code}')
else:
raise ValueError(
f'Widget creation requires name string, application list and a widgetConfig object.')

#clone_dataset.add_widget(token=token, widget_params=widget_payload)
else:
print("Non-rw app. Not cloning.")
except:
except Exception as e:
print(str(e))
raise ValueError(f'Widget cloning failed for {widget.id}')
else:
print("No child widgets to clone!")
vocabs = self.vocabulary
if len(vocabs) > 0:
for v in vocabs:
vocab = v.attributes
if vocab['application']=='rw':
print('Vocab application - ', vocab['application'])
if vocab['application']==source_application:
print(f"Cloning vocab {vocab['name']}")
vocab_payload = {
'application': vocab['application'],
'application': cloned_application,
'name': vocab['name'],
'tags': vocab['tags']
}
try:
clone_dataset.add_vocabulary(vocab_params=vocab_payload, token=token)
except:
raise ValueError('Failed to clone Vocabulary.')
if not dry_run:
try:
clone_dataset.add_vocabulary(vocab_params=vocab_payload, token=token)
except:
raise ValueError('Failed to clone Vocabulary.')
metas = self.metadata
if len(metas) > 0:
for m in metas:
meta = m.attributes
if meta['application']=='rw':
meta_payload = {
"dataset": meta['dataset'],
'application': meta['application'],
'language': meta['language'],
"name": meta['name'],
'description': meta['description'],
"source": meta['source'],
'info': meta['info'],
}
if 'columns' in meta:
meta_payload.update({'columns': meta['columns']})
try:
rw_api_url = 'https://api.resourcewatch.org/v1/dataset/{}/metadata'.format(clone_dataset.id)
res = requests.request("POST", rw_api_url, data=json.dumps(meta_payload),
headers=create_headers())
print('Metadata created.')
except:
raise ValueError('Failed to clone Metadata.')
print('Metadata application - ', meta['application'])
if meta['application']==source_application:
print(f'Cloning metadata {meta["name"]}')
if not dry_run:
meta_payload = {
"dataset": clone_dataset.id,
'application': cloned_application,
'language': meta['language'],
"name": meta['name'],
'description': meta['description'],
"source": meta['source'],
'info': meta['info'],
}
if 'columns' in meta:
meta_payload.update({'columns': meta['columns']})
try:
rw_api_url = 'https://api.resourcewatch.org/v1/dataset/{}/metadata'.format(clone_dataset.id)
headers = {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}
res = requests.request("POST", rw_api_url, data=json.dumps(meta_payload),
headers=headers)
print('Metadata created.')
except:
raise ValueError('Failed to clone Metadata.')
# self.attributes = Dataset(clone_dataset_id, server=clone_server).attributes
return clone_dataset_id
if not dry_run:
return clone_dataset_id
else:
return "Dry run so no cloned dataset ID"

# Make a copy
dataset_to_copy = LMIPy.Dataset(dataset_id)
Expand All @@ -265,4 +290,4 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase
}
# Clone dataset
new_dataset_id = clone_ds(dataset_to_copy, token=API_TOKEN, enviro='production', dataset_params=clone_attributes, clone_children=True, clone_first_layer_only=clone_first_layer_only, clone_default_widget_only=clone_default_widget_only)
print('new dataset API ID:' + new_dataset_id)
print('new dataset API ID:' + new_dataset_id)