-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
HParams: Create hparams data source to fetch data from the hparams pl…
…ugin (#6535) ## Motivation for features / changes Today we fetch data from the hparams plugin using the runs data source. The data is then written to both the runs and hparams state. Unfortunately there are a few issues with both this event structure and our current data model. The data model implies a 1:1 mapping between experiment ids and hparam/metric specs. This is not the case when in an experiment view and thus the model will need to be changed. The event structure is inconsistent with our typical redux structure and thus is hard to refactor to work the way we need it to. This is the first PR in a series I am creating to address this issue. Future PRs will: * add a new effects file, action, and hparams state entries for both a `runToHparamsAndMetrics` mapping along with a single `currentSpecs` (name pending). * remove much of the logic from runs data source and update the `getRuns` selector to populate data ## Screenshots of UI changes (or N/A) N/A
- Loading branch information
1 parent
8616842
commit d3277a4
Showing
12 changed files
with
711 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
198 changes: 198 additions & 0 deletions
198
tensorboard/webapp/hparams/_redux/hparams_data_source.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,198 @@ | ||
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
==============================================================================*/ | ||
import {Injectable} from '@angular/core'; | ||
import {Observable} from 'rxjs'; | ||
import {map} from 'rxjs/operators'; | ||
|
||
import { | ||
Domain, | ||
DomainType, | ||
BackendListSessionGroupRequest, | ||
BackendHparamsExperimentResponse, | ||
BackendHparamSpec, | ||
DiscreteDomainHparamSpec, | ||
SessionGroup, | ||
HparamAndMetricSpec, | ||
IntervalDomainHparamSpec, | ||
BackendListSessionGroupResponse, | ||
RunStatus, | ||
} from '../types'; | ||
import {TBHttpClient} from '../../webapp_data_source/tb_http_client'; | ||
|
||
const HPARAMS_HTTP_PATH_PREFIX = 'data/plugin/hparams'; | ||
|
||
function isHparamDiscrete( | ||
hparam: BackendHparamSpec | ||
): hparam is DiscreteDomainHparamSpec { | ||
return Boolean((hparam as DiscreteDomainHparamSpec).domainDiscrete); | ||
} | ||
|
||
function isHparamInterval( | ||
hparam: BackendHparamSpec | ||
): hparam is IntervalDomainHparamSpec { | ||
return Boolean((hparam as IntervalDomainHparamSpec).domainInterval); | ||
} | ||
|
||
function getHparamDomain(hparam: BackendHparamSpec): Domain { | ||
if (isHparamDiscrete(hparam)) { | ||
return { | ||
type: DomainType.DISCRETE, | ||
values: hparam.domainDiscrete, | ||
}; | ||
} | ||
|
||
if (isHparamInterval(hparam)) { | ||
return { | ||
...hparam.domainInterval, | ||
type: DomainType.INTERVAL, | ||
}; | ||
} | ||
|
||
return { | ||
values: [], | ||
type: DomainType.DISCRETE, | ||
}; | ||
} | ||
|
||
@Injectable() | ||
export class HparamsDataSource { | ||
constructor(private readonly http: TBHttpClient) {} | ||
|
||
private getPrefix(experimentIds: string[]) { | ||
return experimentIds.length > 1 ? 'compare' : 'experiment'; | ||
} | ||
|
||
private formatExperimentIds(experimentIds: string[]) { | ||
if (experimentIds.length === 1) { | ||
return experimentIds[0]; | ||
} | ||
|
||
// The server does not send back experiment ids. Instead the response is formatted as | ||
// `[AliasNumber] ExperimentAlias/RunName` | ||
// By using the index as the alias we can translate associate the response with an experiment id | ||
// Note: The experiment id itself cannot be the alias because it may contain ':' | ||
return experimentIds.map((eid, index) => `${index}:${eid}`).join(','); | ||
} | ||
|
||
fetchExperimentInfo( | ||
experimentIds: string[] | ||
): Observable<HparamAndMetricSpec> { | ||
const formattedExperimentIds = this.formatExperimentIds(experimentIds); | ||
return this.http | ||
.post<BackendHparamsExperimentResponse>( | ||
`/${this.getPrefix( | ||
experimentIds | ||
)}/${formattedExperimentIds}/${HPARAMS_HTTP_PATH_PREFIX}/experiment`, | ||
{experimentName: formattedExperimentIds}, | ||
{}, | ||
'request' | ||
) | ||
.pipe( | ||
map((response) => { | ||
return { | ||
hparams: response.hparamInfos.map((hparam) => { | ||
const feHparam = { | ||
...hparam, | ||
domain: getHparamDomain(hparam), | ||
}; | ||
|
||
delete (feHparam as any).domainInterval; | ||
delete (feHparam as any).domainDiscrete; | ||
|
||
return feHparam; | ||
}), | ||
metrics: response.metricInfos.map((metric) => ({ | ||
...metric, | ||
tag: metric.name.tag, | ||
})), | ||
}; | ||
}) | ||
); | ||
} | ||
|
||
fetchSessionGroups( | ||
experimentIds: string[], | ||
hparamsAndMetricsSpecs: HparamAndMetricSpec | ||
): Observable<SessionGroup[]> { | ||
const formattedExperimentIds = this.formatExperimentIds(experimentIds); | ||
|
||
const colParams: BackendListSessionGroupRequest['colParams'] = []; | ||
|
||
for (const hparam of hparamsAndMetricsSpecs.hparams) { | ||
colParams.push({hparam: hparam.name}); | ||
} | ||
for (const mectric of hparamsAndMetricsSpecs.metrics) { | ||
colParams.push({ | ||
metric: mectric.name, | ||
}); | ||
} | ||
|
||
const listSessionRequestParams: BackendListSessionGroupRequest = { | ||
experimentName: formattedExperimentIds, | ||
allowedStatuses: [ | ||
RunStatus.STATUS_FAILURE, | ||
RunStatus.STATUS_RUNNING, | ||
RunStatus.STATUS_SUCCESS, | ||
RunStatus.STATUS_UNKNOWN, | ||
], | ||
colParams, | ||
startIndex: 0, | ||
// arbitrary large number so it does not get clipped. | ||
sliceSize: 1e6, | ||
}; | ||
|
||
return this.http | ||
.post<BackendListSessionGroupResponse>( | ||
`/${this.getPrefix( | ||
experimentIds | ||
)}/${formattedExperimentIds}/${HPARAMS_HTTP_PATH_PREFIX}/session_groups`, | ||
listSessionRequestParams, | ||
{}, | ||
'request' | ||
) | ||
.pipe( | ||
map((response) => | ||
response.sessionGroups.map((sessionGroup) => { | ||
sessionGroup.sessions = sessionGroup.sessions.map((session) => { | ||
/* | ||
* In single experiment mode the Session.name is equal to the runName. | ||
* In comparison view it is `[AliasNumber] ExperimentAlias/runName` | ||
* | ||
* We store runs as experimentId/runName so it is necessary to prepend the experiment name | ||
* in single experiment view. "In comparison view we pass the indeces of the experimentIds | ||
* as the aliases in the request. That allows us to parse the indeces from the response and | ||
* use them to lookup the correct ids from the experimentIds argument. | ||
*/ | ||
if (experimentIds.length > 1) { | ||
const [, ...aliasAndRunName] = session.name.split(' '); | ||
const [experimentIndex, ...runName] = aliasAndRunName | ||
.join(' ') | ||
.split('/'); | ||
session.name = [ | ||
// This parseInt should not be necessary because JS Arrays DO support indexing by string | ||
experimentIds[parseInt(experimentIndex)], | ||
...runName, | ||
].join('/'); | ||
} else { | ||
session.name = [experimentIds[0], session.name].join('/'); | ||
} | ||
return session; | ||
}); | ||
return sessionGroup; | ||
}) | ||
) | ||
); | ||
} | ||
} |
Oops, something went wrong.