Skip to content

Hparams: Change some handling/generation of hparams with discrete domains. #6489

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 13, 2023
Merged

Hparams: Change some handling/generation of hparams with discrete domains. #6489

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ec90f5d
Change some handling/generation of hparams with discrete domains.
bmd3k Jul 11, 2023
b99d5e1
Format.
bmd3k Jul 11, 2023
9e280c7
Add a space.
bmd3k Jul 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions tensorboard/plugins/hparams/api.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,7 @@ message HParamInfo {
// every instance of this hyperparameter will hold a value from this set. It
// is used by the UI to allow filtering so that only session groups (see
// below) whose associated hyperparameter value "passes" the filter are
// displayed. If this is not populated, the domain is assumed to be the
// entire domain of the type of the hyperparameter.
// displayed. If this is not populated, the domain is assumed to be empty.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure I understand this what is meant by an empty domain. I would expect this to always be populated.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's possible, for example, in hparam summary api to specify that an hparam exists but not provide any values for it for any of the runs.

oneof domain {
// A discrete set of the values this hyperparameter can hold.
google.protobuf.ListValue domain_discrete = 5;
Expand Down
17 changes: 3 additions & 14 deletions tensorboard/plugins/hparams/backend_context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,18 +44,13 @@ class Context:
no better place. See http://wiki.c2.com/?MagicContainer
"""

def __init__(self, tb_context, max_domain_discrete_len=10):
def __init__(self, tb_context):
"""Instantiates a context.

Args:
tb_context: base_plugin.TBContext. The "base" context we extend.
max_domain_discrete_len: int. Only used when computing the experiment
from the session runs. The maximum number of disticnt values a string
hyperparameter can have for us to populate its 'domain_discrete' field.
Typically, only tests should specify a value for this parameter.
"""
self._tb_context = tb_context
self._max_domain_discrete_len = max_domain_discrete_len

def experiment_from_metadata(
self,
Expand Down Expand Up @@ -245,8 +240,7 @@ def _compute_hparam_infos(self, hparams_run_to_tag_to_content):
Finds all the SessionStartInfo messages and collects the hparams values
appearing in each one. For each hparam attempts to deduce a type that fits
all its values. Finally, sets the 'domain' of the resulting HParamInfo
to be discrete if the type is string and the number of distinct values is
small enough.
to be discrete if the type is string or boolean.

Returns:
A list of api_pb2.HParamInfo messages.
Expand Down Expand Up @@ -309,12 +303,7 @@ def _compute_hparam_info_from_values(self, name, values):
if result.type == api_pb2.DATA_TYPE_UNSET:
return None

# If the result is a string, set the domain to be the distinct values if
# there aren't too many of them.
if (
result.type == api_pb2.DATA_TYPE_STRING
and len(distinct_values) <= self._max_domain_discrete_len
):
if result.type == api_pb2.DATA_TYPE_STRING:
result.domain_discrete.extend(distinct_values)

if result.type == api_pb2.DATA_TYPE_BOOL:
Expand Down
58 changes: 2 additions & 56 deletions tensorboard/plugins/hparams/backend_context_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,11 +152,9 @@ def _mock_list_hyperparameters(
):
return self._hyperparameters

def _experiment_from_metadata(self, max_domain_discrete_len=10):
def _experiment_from_metadata(self):
"""Calls the expected operations for generating an Experiment proto."""
ctxt = backend_context.Context(
self._mock_tb_context, max_domain_discrete_len
)
ctxt = backend_context.Context(self._mock_tb_context)
request_ctx = context.RequestContext()
return ctxt.experiment_from_metadata(
request_ctx,
Expand Down Expand Up @@ -306,58 +304,6 @@ def test_experiment_without_experiment_tag_different_hparam_types(self):
_canonicalize_experiment(actual_exp)
self.assertProtoEquals(expected_exp, actual_exp)

def test_experiment_without_experiment_tag_many_distinct_values(self):
self.session_1_start_info_ = """
hparams:[
{key: 'batch_size' value: {number_value: 100}},
{key: 'lr' value: {string_value: '0.01'}}
]
"""
self.session_2_start_info_ = """
hparams:[
{key: 'lr' value: {number_value: 0.02}},
{key: 'model_type' value: {string_value: 'CNN'}}
]
"""
self.session_3_start_info_ = """
hparams:[
{key: 'batch_size' value: {bool_value: true}},
{key: 'model_type' value: {string_value: 'CNN'}}
]
"""
expected_exp = """
hparam_infos: {
name: 'batch_size'
type: DATA_TYPE_STRING
}
hparam_infos: {
name: 'lr'
type: DATA_TYPE_STRING
}
hparam_infos: {
name: 'model_type'
type: DATA_TYPE_STRING
domain_discrete: {
values: [{string_value: 'CNN'}]
}
}
metric_infos: {
name: {group: '', tag: 'accuracy'}
}
metric_infos: {
name: {group: '', tag: 'loss'}
}
metric_infos: {
name: {group: 'eval', tag: 'loss'}
}
metric_infos: {
name: {group: 'train', tag: 'loss'}
}
"""
actual_exp = self._experiment_from_metadata(max_domain_discrete_len=1)
_canonicalize_experiment(actual_exp)
self.assertProtoEquals(expected_exp, actual_exp)

def test_experiment_with_bool_types(self):
self.session_1_start_info_ = """
hparams:[
Expand Down
46 changes: 25 additions & 21 deletions tensorboard/plugins/hparams/tf_hparams_query_pane/tf-hparams-query-pane.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ interface ColumnMetric {
order?: string;
}

const MAX_DOMAIN_DISCRETE_LIST_LEN = 10;

/**
* The tf-hparams-query-pane element implements controls for querying the
* server for a list of session groups. It provides filtering, and
Expand Down Expand Up @@ -102,7 +104,7 @@ class TfHparamsQueryPane extends LegacyElementMixin(PolymerElement) {
</paper-input>
</template>
<!-- 3. A regexp -->
<template is="dom-if" if="[[hparam.filter.regexp]]">
<template is="dom-if" if="[[_hasRegexpFilter(hparam)]]">
<paper-input
label="Regular expression"
value="{{hparam.filter.regexp}}"
Expand Down Expand Up @@ -581,25 +583,22 @@ class TfHparamsQueryPane extends LegacyElementMixin(PolymerElement) {
filter: {} as any,
};
if (hparam.info.hasOwnProperty('domainDiscrete')) {
hparam.filter.domainDiscrete = [];
hparam.info.domainDiscrete.forEach((val) => {
hparam.filter.domainDiscrete.push({
value: val,
checked: true,
// Handle a discrete domain. Could be of any data type.
if (hparam.info.domainDiscrete.length < MAX_DOMAIN_DISCRETE_LIST_LEN) {
hparam.filter.domainDiscrete = [];
hparam.info.domainDiscrete.forEach((val: any) => {
hparam.filter.domainDiscrete.push({
value: val,
checked: true,
});
});
});
} else if (hparam.info.type === 'DATA_TYPE_BOOL') {
hparam.filter.domainDiscrete = [
{
value: false,
checked: true,
},
{
value: true,
checked: true,
},
];
} else {
// Don't show long lists of values. If the list surpasses a certain
// threshold then the user instead specifies regex filters.
hparam.filter.regexp = '';
}
} else if (hparam.info.type === 'DATA_TYPE_FLOAT64') {
// Handle a float interval domain.
hparam.filter.interval = {
min: {
value: '',
Expand All @@ -610,10 +609,11 @@ class TfHparamsQueryPane extends LegacyElementMixin(PolymerElement) {
invalid: false,
},
};
} else if (hparam.info.type === 'DATA_TYPE_STRING') {
hparam.filter.regexp = '';
} else {
console.warn('unknown hparam.info.type: %s', hparam.info.type);
console.warn(
'cannot process domain type %s without discrete domain values',
hparam.info.type
);
}
result.push(hparam);
});
Expand Down Expand Up @@ -690,6 +690,10 @@ class TfHparamsQueryPane extends LegacyElementMixin(PolymerElement) {
metricInfos: newMetricInfos,
};
}
// Determines if a regex filter should be rendered.
_hasRegexpFilter(hparam) {
return hparam.filter.regexp !== undefined;
}
// Sends a query to the server for the list of session groups.
// Asynchronously updates the sessionGroups property with the response.
_queryServer() {
Expand Down