Skip to content

Commit

Permalink
HParams: Create hparams data source to fetch data from the hparams pl…
Browse files Browse the repository at this point in the history
…ugin (#6535)

## Motivation for features / changes
Today we fetch data from the hparams plugin using the runs data source.
The data is then written to both the runs and hparams state.
Unfortunately there are a few issues with both this event structure and
our current data model.

The data model implies a 1:1 mapping between experiment ids and
hparam/metric specs. This is not the case when in an experiment view and
thus the model will need to be changed.

The event structure is inconsistent with our typical redux structure and
thus is hard to refactor to work the way we need it to.

This is the first PR in a series I am creating to address this issue.
Future PRs will:
* add a new effects file, action, and hparams state entries for both a
`runToHparamsAndMetrics` mapping along with a single `currentSpecs`
(name pending).
* remove much of the logic from runs data source and update the
`getRuns` selector to populate data

## Screenshots of UI changes (or N/A)
N/A
  • Loading branch information
rileyajones authored Aug 10, 2023
1 parent 8616842 commit d3277a4
Show file tree
Hide file tree
Showing 12 changed files with 711 additions and 5 deletions.
1 change: 1 addition & 0 deletions tensorboard/webapp/hparams/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ tf_ts_library(
],
deps = [
"//tensorboard/webapp/runs/data_source",
"//tensorboard/webapp/runs/data_source:backend_types",
],
)

Expand Down
18 changes: 18 additions & 0 deletions tensorboard/webapp/hparams/_redux/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,20 @@ tf_ts_library(
],
)

tf_ng_module(
name = "hparams_data_source",
srcs = [
"hparams_data_source.ts",
],
deps = [
"//tensorboard/webapp/hparams:types",
"//tensorboard/webapp/webapp_data_source:http_client",
"@npm//@angular/core",
"@npm//@ngrx/store",
"@npm//rxjs",
],
)

tf_ts_library(
name = "testing",
testonly = True,
Expand All @@ -98,21 +112,25 @@ tf_ts_library(
name = "_redux_test_lib",
testonly = True,
srcs = [
"hparams_data_source_test.ts",
"hparams_reducers_test.ts",
"hparams_selectors_test.ts",
"hparams_selectors_utils_test.ts",
"utils_test.ts",
],
deps = [
":hparams_actions",
":hparams_data_source",
":hparams_reducers",
":hparams_selectors",
":testing",
":utils",
"//tensorboard/webapp/angular:expect_angular_core_testing",
"//tensorboard/webapp/hparams:types",
"//tensorboard/webapp/runs/actions",
"//tensorboard/webapp/runs/data_source:testing",
"//tensorboard/webapp/runs/store:testing",
"//tensorboard/webapp/webapp_data_source:http_client_testing",
"@npm//@types/jasmine",
],
)
198 changes: 198 additions & 0 deletions tensorboard/webapp/hparams/_redux/hparams_data_source.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
import {Injectable} from '@angular/core';
import {Observable} from 'rxjs';
import {map} from 'rxjs/operators';

import {
Domain,
DomainType,
BackendListSessionGroupRequest,
BackendHparamsExperimentResponse,
BackendHparamSpec,
DiscreteDomainHparamSpec,
SessionGroup,
HparamAndMetricSpec,
IntervalDomainHparamSpec,
BackendListSessionGroupResponse,
RunStatus,
} from '../types';
import {TBHttpClient} from '../../webapp_data_source/tb_http_client';

const HPARAMS_HTTP_PATH_PREFIX = 'data/plugin/hparams';

function isHparamDiscrete(
hparam: BackendHparamSpec
): hparam is DiscreteDomainHparamSpec {
return Boolean((hparam as DiscreteDomainHparamSpec).domainDiscrete);
}

function isHparamInterval(
hparam: BackendHparamSpec
): hparam is IntervalDomainHparamSpec {
return Boolean((hparam as IntervalDomainHparamSpec).domainInterval);
}

function getHparamDomain(hparam: BackendHparamSpec): Domain {
if (isHparamDiscrete(hparam)) {
return {
type: DomainType.DISCRETE,
values: hparam.domainDiscrete,
};
}

if (isHparamInterval(hparam)) {
return {
...hparam.domainInterval,
type: DomainType.INTERVAL,
};
}

return {
values: [],
type: DomainType.DISCRETE,
};
}

@Injectable()
export class HparamsDataSource {
constructor(private readonly http: TBHttpClient) {}

private getPrefix(experimentIds: string[]) {
return experimentIds.length > 1 ? 'compare' : 'experiment';
}

private formatExperimentIds(experimentIds: string[]) {
if (experimentIds.length === 1) {
return experimentIds[0];
}

// The server does not send back experiment ids. Instead the response is formatted as
// `[AliasNumber] ExperimentAlias/RunName`
// By using the index as the alias we can translate associate the response with an experiment id
// Note: The experiment id itself cannot be the alias because it may contain ':'
return experimentIds.map((eid, index) => `${index}:${eid}`).join(',');
}

fetchExperimentInfo(
experimentIds: string[]
): Observable<HparamAndMetricSpec> {
const formattedExperimentIds = this.formatExperimentIds(experimentIds);
return this.http
.post<BackendHparamsExperimentResponse>(
`/${this.getPrefix(
experimentIds
)}/${formattedExperimentIds}/${HPARAMS_HTTP_PATH_PREFIX}/experiment`,
{experimentName: formattedExperimentIds},
{},
'request'
)
.pipe(
map((response) => {
return {
hparams: response.hparamInfos.map((hparam) => {
const feHparam = {
...hparam,
domain: getHparamDomain(hparam),
};

delete (feHparam as any).domainInterval;
delete (feHparam as any).domainDiscrete;

return feHparam;
}),
metrics: response.metricInfos.map((metric) => ({
...metric,
tag: metric.name.tag,
})),
};
})
);
}

fetchSessionGroups(
experimentIds: string[],
hparamsAndMetricsSpecs: HparamAndMetricSpec
): Observable<SessionGroup[]> {
const formattedExperimentIds = this.formatExperimentIds(experimentIds);

const colParams: BackendListSessionGroupRequest['colParams'] = [];

for (const hparam of hparamsAndMetricsSpecs.hparams) {
colParams.push({hparam: hparam.name});
}
for (const mectric of hparamsAndMetricsSpecs.metrics) {
colParams.push({
metric: mectric.name,
});
}

const listSessionRequestParams: BackendListSessionGroupRequest = {
experimentName: formattedExperimentIds,
allowedStatuses: [
RunStatus.STATUS_FAILURE,
RunStatus.STATUS_RUNNING,
RunStatus.STATUS_SUCCESS,
RunStatus.STATUS_UNKNOWN,
],
colParams,
startIndex: 0,
// arbitrary large number so it does not get clipped.
sliceSize: 1e6,
};

return this.http
.post<BackendListSessionGroupResponse>(
`/${this.getPrefix(
experimentIds
)}/${formattedExperimentIds}/${HPARAMS_HTTP_PATH_PREFIX}/session_groups`,
listSessionRequestParams,
{},
'request'
)
.pipe(
map((response) =>
response.sessionGroups.map((sessionGroup) => {
sessionGroup.sessions = sessionGroup.sessions.map((session) => {
/*
* In single experiment mode the Session.name is equal to the runName.
* In comparison view it is `[AliasNumber] ExperimentAlias/runName`
*
* We store runs as experimentId/runName so it is necessary to prepend the experiment name
* in single experiment view. "In comparison view we pass the indeces of the experimentIds
* as the aliases in the request. That allows us to parse the indeces from the response and
* use them to lookup the correct ids from the experimentIds argument.
*/
if (experimentIds.length > 1) {
const [, ...aliasAndRunName] = session.name.split(' ');
const [experimentIndex, ...runName] = aliasAndRunName
.join(' ')
.split('/');
session.name = [
// This parseInt should not be necessary because JS Arrays DO support indexing by string
experimentIds[parseInt(experimentIndex)],
...runName,
].join('/');
} else {
session.name = [experimentIds[0], session.name].join('/');
}
return session;
});
return sessionGroup;
})
)
);
}
}
Loading

0 comments on commit d3277a4

Please sign in to comment.