Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unified Queue: fix failing tests and address TODOs #26067

Merged
merged 15 commits into from
Feb 5, 2025
12 changes: 10 additions & 2 deletions server/datastore/mysql/activities.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,15 @@ func (ds *Datastore) NewActivity(
cols = append(cols, "user_email")
}

if vppAct, ok := activity.(fleet.ActivityInstalledAppStoreApp); ok {
vppPtrAct, okPtr := activity.(*fleet.ActivityInstalledAppStoreApp)
vppAct, ok := activity.(fleet.ActivityInstalledAppStoreApp)
if okPtr || ok {
hostID := vppAct.HostID
cmdUUID := vppAct.CommandUUID
if okPtr {
cmdUUID = vppPtrAct.CommandUUID
hostID = vppPtrAct.HostID
}
// NOTE: ideally this would be called in the same transaction as storing
// the nanomdm command results, but the current design doesn't allow for
// that with the nano store being a distinct entity to our datastore (we
Expand All @@ -76,7 +84,7 @@ func (ds *Datastore) NewActivity(
// gets created only when the MDM command status is in a final state
// (success or failure), which is exactly when we want to activate the next
// activity.
if _, err := ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), vppAct.HostID, vppAct.CommandUUID); err != nil {
if _, err := ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), hostID, cmdUUID); err != nil {
return ctxerr.Wrap(ctx, err, "activate next activity from VPP app install")
}
}
Expand Down
2 changes: 1 addition & 1 deletion server/datastore/mysql/labels.go
Original file line number Diff line number Diff line change
Expand Up @@ -695,7 +695,7 @@ func (ds *Datastore) applyHostLabelFilters(ctx context.Context, filter fleet.Tea
return "", nil, ctxerr.Wrap(ctx, err, "get software installer metadata by team and title id")

default:
// TODO(uniq): prior code was joining on installer id but based on how list options are parsed [1] it seems like this should be the title id
// TODO(Sarah): prior code was joining on installer id but based on how list options are parsed [1] it seems like this should be the title id
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gillespi314 I changed the TODO to you since it's not related to the unified queue, let me know if that's ok with you (or if you want me to just remove it)?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's change it to a FIXME and I'll flag it for the g-software folks to take a closer look.

Copy link
Member Author

@mna mna Feb 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On second thought, is it really "to fix" or is your change the actual fix? I.e. is it just to raise awareness that this was changed, or does that current implementation need to be changed? I'd remove the comment altogether if there's not really anything to fix.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mna, you're right, it was more so to raise awareness of the change in case I might have missed something.

@jahzielv, do you mind giving this a quick sanity check? There's a similar note regarding VPP to check here.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gillespi314 so sorry, I totally missed this notification! I can take a look

// [1] https://github.com/fleetdm/fleet/blob/8aecae4d853829cb6e7f828099a4f0953643cf18/server/datastore/mysql/hosts.go#L1088-L1089
installerJoin, installerParams, err := ds.softwareInstallerJoin(*opt.SoftwareTitleIDFilter, *opt.SoftwareStatusFilter)
if err != nil {
Expand Down
32 changes: 15 additions & 17 deletions server/datastore/mysql/scripts.go
Original file line number Diff line number Diff line change
Expand Up @@ -276,11 +276,13 @@ func (ds *Datastore) ListReadyToExecuteScriptsForHost(ctx context.Context, hostI
func (ds *Datastore) listUpcomingHostScriptExecutions(ctx context.Context, hostID uint, onlyShowInternal, onlyReadyToExecute bool) ([]*fleet.HostScriptResult, error) {
extraWhere := ""
if onlyShowInternal {
extraWhere = " AND ua.payload->'$.is_internal' = 1"
// software_uninstalls are implicitly internal
extraWhere = " AND COALESCE(ua.payload->'$.is_internal', 1) = 1"
}
if onlyReadyToExecute {
extraWhere += " AND ua.activated_at IS NOT NULL"
}
// this selects software uninstalls too as they run as scripts
listStmt := fmt.Sprintf(`
SELECT
id,
Expand All @@ -299,21 +301,17 @@ func (ds *Datastore) listUpcomingHostScriptExecutions(ctx context.Context, hostI
IF(ua.activated_at IS NULL, 0, 1) AS topmost
FROM
upcoming_activities ua
INNER JOIN script_upcoming_activities sua
-- left join because software_uninstall has no script join
LEFT JOIN script_upcoming_activities sua
ON ua.id = sua.upcoming_activity_id
WHERE
ua.host_id = ? AND
ua.activity_type = 'script' AND
(
ua.payload->'$.sync_request' = 0 OR
ua.created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sync scripts are treated as any other script in the upcoming queue: #22866 (comment)

)
ua.activity_type IN ('script', 'software_uninstall')
%s
ORDER BY topmost DESC, priority DESC, created_at ASC) t`, extraWhere)

var results []*fleet.HostScriptResult
seconds := int(constants.MaxServerWaitTime.Seconds())
if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, listStmt, hostID, seconds); err != nil {
if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, listStmt, hostID); err != nil {
return nil, ctxerr.Wrap(ctx, err, "list pending host script executions")
}
return results, nil
Expand Down Expand Up @@ -911,7 +909,7 @@ WHERE
INNER JOIN script_upcoming_activities sua
ON upcoming_activities.id = sua.upcoming_activity_id
WHERE
upcoming_activities.activity_type = 'script'
upcoming_activities.activity_type = 'script'
AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND)
AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)`

Expand All @@ -932,13 +930,13 @@ WHERE
exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND)
AND script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))`

const clearPendingExecutionsNotInListUA = `DELETE FROM upcoming_activities
USING
const clearPendingExecutionsNotInListUA = `DELETE FROM upcoming_activities
USING
upcoming_activities
INNER JOIN script_upcoming_activities sua
ON upcoming_activities.id = sua.upcoming_activity_id
WHERE
upcoming_activities.activity_type = 'script'
upcoming_activities.activity_type = 'script'
AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND)
AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))`

Expand All @@ -957,13 +955,13 @@ ON DUPLICATE KEY UPDATE
exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND)
AND script_id = ? AND script_content_id != ?`

const clearPendingExecutionsWithObsoleteScriptUA = `DELETE FROM upcoming_activities
USING
upcoming_activities
const clearPendingExecutionsWithObsoleteScriptUA = `DELETE FROM upcoming_activities
USING
upcoming_activities
INNER JOIN script_upcoming_activities sua
ON upcoming_activities.id = sua.upcoming_activity_id
WHERE
upcoming_activities.activity_type = 'script'
upcoming_activities.activity_type = 'script'
AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND)
AND sua.script_id = ? AND sua.script_content_id != ?`

Expand Down
12 changes: 7 additions & 5 deletions server/datastore/mysql/scripts_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,17 +230,19 @@ func testHostScriptResult(t *testing.T, ds *Datastore) {
require.Equal(t, createdScript3.ID, pending[0].ID)

// modify the upcoming script to be a sync script that has
// been pending for a long time
// been pending for a long time doesn't change result
// https://github.com/fleetdm/fleet/issues/22866#issuecomment-2575961141
ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error {
_, err := tx.ExecContext(ctx, "UPDATE upcoming_activities SET created_at = ?, payload = JSON_SET(payload, '$.sync_request', true) WHERE id = ?",
time.Now().Add(-24*time.Hour), createdScript3.ID)
_, err := tx.ExecContext(ctx, "UPDATE upcoming_activities SET created_at = ?, payload = JSON_SET(payload, '$.sync_request', ?) WHERE id = ?",
time.Now().Add(-24*time.Hour), true, createdScript3.ID)
return err
})

// the script is not pending anymore
// the script is still pending
pending, err = ds.ListPendingHostScriptExecutions(ctx, 1, false)
require.NoError(t, err)
require.Len(t, pending, 0)
require.Len(t, pending, 1)
require.Equal(t, createdScript3.ExecutionID, pending[0].ExecutionID)

// check that scripts with large unsigned error codes get
// converted to signed error codes
Expand Down
8 changes: 4 additions & 4 deletions server/datastore/mysql/software_installers.go
Original file line number Diff line number Diff line change
Expand Up @@ -1089,7 +1089,7 @@ WHERE
func (ds *Datastore) GetSummaryHostSoftwareInstalls(ctx context.Context, installerID uint) (*fleet.SoftwareInstallerStatusSummary, error) {
var dest fleet.SoftwareInstallerStatusSummary

// TODO(uniq): AFAICT we don't have uniqueness for host_id + title_id in upcoming or
// TODO(Sarah): AFAICT we don't have uniqueness for host_id + title_id in upcoming or
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here and a couple more todos below @gillespi314 I changed the TODO to you since it's not related to the unified queue, let me know if that's ok with you (or if you want me to just remove it)? The load tests should (hopefully) tell us if we need to improve some queries.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one seems a bit more related to UniQ insofar we probably want the ordering of activities to be consistent with other queries. What if we change these to use the same group-wise max approach we're using elsewhere?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah I see, yeah I think once the row is in host_software_installs or host_script_results, it doesn't matter much (the MAX(id) is as good as before) because the upcoming activities will create those rows in the correct order, it matters more when querying the upcoming_activities. Also, while looking at those uniqueness/groupwise max comments, I think I may have found an issue in the software status queries?

When the filter is for non-pending, we do this:

// for non-pending statuses, we'll join through host_software_installs filtered by the status
statusFilter := "hsi.status = :status"
if status == fleet.SoftwareFailed {
// failed is a special case, we must include both install and uninstall failures
statusFilter = "hsi.status IN (:installFailed, :uninstallFailed)"
}
// TODO(Sarah): AFAICT we don't have uniqueness for host_id + title_id in upcoming or
// past activities. In the past the max(id) approach was "good enough" as a proxy for the most
// recent activity since we didn't really need to worry about the order of activities.
// Moving to a time-based approach would be more accurate but would require a more complex and
// potentially slower query.
stmt := fmt.Sprintf(`JOIN (
SELECT
host_id
FROM
host_software_installs hsi
WHERE
software_title_id = :title_id
AND hsi.id IN(
SELECT
max(id) -- ensure we use only the most recent install attempt for each host
FROM host_software_installs
WHERE
host_id = hsi.host_id
AND software_title_id = :title_id
AND removed = 0
GROUP BY
host_id, software_title_id)
AND %s) hss ON hss.host_id = h.id
`, statusFilter)

But I think it should not consider any of those rows if there's also a pending request in upcoming_activities, right? I.e. if a software has an installed and a failed status in host_software_installs and a pending one in upcoming_activities, the only status that matters for the filter is the latest, so it is pending?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I'll address both points in a subsequent PR (use the left join approach for the groupwise max, and fix the status filter). Let me know if I'm mistaken and it's current behavior is correct, though!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But I think it should not consider any of those rows if there's also a pending request in upcoming_activities, right?

Doesn't the early return at line 1271 cover it?

Or are you saying that when filtering on a terminal status (e.g., installed), if a host has matching record for a prior install the host should always be excluded from the results if there's an entry in upcoming activities? TBH I'm not very clear what is expected in that case. What you're suggesting makes sense if pending is always supposed to take precedence of prior activity. Does it depend on the specific use case (i.e. for "last install" we always give preference to upcoming, but maybe something else when filtering for a specific status)?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah the latter, if I filter on installed but the software has a pending entry in addition to a previous install, I think it's latest status is the one we want to filter on, so it should not be returned.

That's how it worked before, it always considers the latest install attempt, regardless of status:
https://github.com/fleetdm/fleet/blob/main/server/datastore/mysql/software_installers.go#L997-L1014

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh wait, no I misread that... I think it does return the host if it has the requested status, it's just that we take the latest entry with that status.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah no, sorry again :D It's as I said before, it only considers the latest attempt, and then it returns the host if that attempt's status is the one requested by the filter. A bit hairy but yeah, that's the behavior to reproduce so I'll update that query accordingly.

// past activities. In the past the max(id) approach was "good enough" as a proxy for the most
// recent activity since we didn't really need to worry about the order of activities.
// Moving to a time-based approach would be more accurate but would require a more complex and
Expand Down Expand Up @@ -1207,7 +1207,7 @@ WHERE
status = fleet.SoftwareInstallFailed // TODO: When VPP supports uninstall this should become STATUS IN ('failed_install', 'failed_uninstall')
}

// TODO(uniq): AFAICT we don't have uniqueness for host_id + title_id in upcoming or
// TODO(Sarah): AFAICT we don't have uniqueness for host_id + title_id in upcoming or
// past activities. In the past the max(id) approach was "good enough" as a proxy for the most
// recent activity since we didn't really need to worry about the order of activities.
// Moving to a time-based approach would be more accurate but would require a more complex and
Expand Down Expand Up @@ -1278,7 +1278,7 @@ WHERE
statusFilter = "hsi.status IN (:installFailed, :uninstallFailed)"
}

// TODO(uniq): AFAICT we don't have uniqueness for host_id + title_id in upcoming or
// TODO(Sarah): AFAICT we don't have uniqueness for host_id + title_id in upcoming or
// past activities. In the past the max(id) approach was "good enough" as a proxy for the most
// recent activity since we didn't really need to worry about the order of activities.
// Moving to a time-based approach would be more accurate but would require a more complex and
Expand Down Expand Up @@ -1308,7 +1308,7 @@ WHERE
"status": status,
"installFailed": fleet.SoftwareInstallFailed,
"uninstallFailed": fleet.SoftwareUninstallFailed,
// TODO(uniq): prior code was joining based on installer id but based on how list options are parsed [1] it seems like this should be the title id
// TODO(Sarah): prior code was joining based on installer id but based on how list options are parsed [1] it seems like this should be the title id
// [1] https://github.com/fleetdm/fleet/blob/8aecae4d853829cb6e7f828099a4f0953643cf18/server/datastore/mysql/hosts.go#L1088-L1089
"title_id": titleID,
})
Expand Down
2 changes: 1 addition & 1 deletion server/datastore/mysql/software_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6000,7 +6000,7 @@ func testDeletedInstalledSoftware(t *testing.T, ds *Datastore) {
ValidatedLabels: &fleet.LabelIdentsWithScope{},
})
require.NoError(t, err)
_, err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID, false, nil)
_, err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID, fleet.HostSoftwareInstallOptions{})
require.NoError(t, err)

ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
Expand Down
7 changes: 5 additions & 2 deletions server/datastore/mysql/vpp.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,9 @@ func (ds *Datastore) GetSummaryHostVPPAppInstalls(ctx context.Context, teamID *u
// not handling it as part of the unified queue work.

// TODO(uniq): refactor vppHostStatusNamedQuery to use the same logic as below
// re: ^ : seems like the logic is the same except for pending/null?

// TODO(uniq): AFAICT we don't have uniqueness for host_id + title_id in upcoming or
// TODO(Sarah): AFAICT we don't have uniqueness for host_id + title_id in upcoming or
// past activities. In the past the max(id) approach was "good enough" as a proxy for the most
// recent activity since we didn't really need to worry about the order of activities.
// Moving to a time-based approach would be more accurate but would require a more complex and
Expand Down Expand Up @@ -180,6 +181,8 @@ SELECT
WHEN ncr.status = :mdm_status_error OR ncr.status = :mdm_status_format_error THEN
:software_status_failed
ELSE
-- TODO should that count as pending install (should be covered by the upcoming
-- case, but seems more correct to report as pending it is found in past)
NULL -- either pending or not installed via VPP App
END AS status
FROM
Expand Down Expand Up @@ -706,7 +709,7 @@ func (ds *Datastore) GetVPPAppMetadataByAdamIDPlatformTeamID(ctx context.Context
vat.self_service,
va.title_id,
va.platform,
va.created_at,
va.created_at,
va.updated_at,
vat.id
FROM vpp_apps va
Expand Down
2 changes: 1 addition & 1 deletion server/fleet/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,7 @@ const (
RunScriptHostOfflineErrMsg = "Script can't run on offline host."
RunScriptForbiddenErrMsg = "You don't have the right permissions in Fleet to run the script."
RunScriptAlreadyRunningErrMsg = "A script is already running on this host. Please wait about 5 minutes to let it finish."
RunScriptHostTimeoutErrMsg = "Fleet didn't hear back from the host in under 5 minutes (timeout for live scripts). Fleet doesn't know if the script ran because it didn't receive the result. Please try again."
RunScriptHostTimeoutErrMsg = "Fleet didn't hear back from the host in under 5 minutes (timeout for live scripts). Fleet doesn't know if the script ran because it didn't receive the result. Go to Fleet and check Host details > Activities to see script results."
RunScriptScriptsDisabledGloballyErrMsg = "Running scripts is disabled in organization settings."
RunScriptDisabledErrMsg = "Scripts are disabled for this host. To run scripts, deploy the fleetd agent with scripts enabled."
RunScriptsOrbitDisabledErrMsg = "Couldn't run script. To run a script, deploy the fleetd agent with --enable-scripts."
Expand Down
31 changes: 17 additions & 14 deletions server/service/integration_core_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12218,16 +12218,19 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() {
h1Foo, err := s.ds.InsertSoftwareInstallRequest(ctx, host1.ID, s1Meta.InstallerID, fleet.HostSoftwareInstallOptions{})
require.NoError(t, err)

// force an order to the activities
endTime := mysql.SetOrderedCreatedAtTimestamps(t, s.ds, time.Now(), "host_script_results", "execution_id", h1A, h1B)
endTime = mysql.SetOrderedCreatedAtTimestamps(t, s.ds, endTime, "host_software_installs", "execution_id", h1Foo)
mysql.SetOrderedCreatedAtTimestamps(t, s.ds, endTime, "host_script_results", "execution_id", h1C, h1D, h1E)
// force an order to the activities (h1A must be first as it was automatically activated)
mysql.SetOrderedCreatedAtTimestamps(t, s.ds, time.Now(), "upcoming_activities", "execution_id", h1A, h1B, h1Foo, h1C, h1D, h1E)

// modify the timestamp h1A and h1B to simulate an script that has been
// pending for a long time (h1A is a sync request, so it will be ignored for
// upcoming activities)
// pending for a long time (h1A is a sync script but that doesn't change
// anything anymore, sync scripts are part of the queue like any other:
// https://github.com/fleetdm/fleet/issues/22866#issuecomment-2575961141)
mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error {
_, err := tx.ExecContext(ctx, "UPDATE host_script_results SET created_at = ? WHERE execution_id IN (?, ?)", time.Now().Add(-24*time.Hour), h1A, h1B)
_, err := tx.ExecContext(ctx, "UPDATE upcoming_activities SET created_at = ? WHERE execution_id = ?", time.Now().Add(-24*time.Hour), h1A)
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, "UPDATE upcoming_activities SET created_at = ? WHERE execution_id = ?", time.Now().Add(-23*time.Hour), h1B)
return err
})

Expand All @@ -12237,22 +12240,22 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() {
wantMeta *fleet.PaginationMetadata
}{
{
wantExecs: []string{h1B, h1Foo, h1C, h1D, h1E},
wantExecs: []string{h1A, h1B, h1Foo, h1C, h1D, h1E},
wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: false},
},
{
queries: []string{"per_page", "2"},
wantExecs: []string{h1B, h1Foo},
wantExecs: []string{h1A, h1B},
wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false},
},
{
queries: []string{"per_page", "2", "page", "1"},
wantExecs: []string{h1C, h1D},
wantExecs: []string{h1Foo, h1C},
wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true},
},
{
queries: []string{"per_page", "2", "page", "2"},
wantExecs: []string{h1E},
wantExecs: []string{h1D, h1E},
wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true},
},
{
Expand All @@ -12262,12 +12265,12 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() {
},
{
queries: []string{"per_page", "3"},
wantExecs: []string{h1B, h1Foo, h1C},
wantExecs: []string{h1A, h1B, h1Foo},
wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false},
},
{
queries: []string{"per_page", "3", "page", "1"},
wantExecs: []string{h1D, h1E},
wantExecs: []string{h1C, h1D, h1E},
wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true},
},
{
Expand All @@ -12282,7 +12285,7 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() {
queryArgs := c.queries
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/activities/upcoming", host1.ID), nil, http.StatusOK, &listResp, queryArgs...)

require.Equal(t, uint(5), listResp.Count)
require.Equal(t, uint(6), listResp.Count)
require.Equal(t, len(c.wantExecs), len(listResp.Activities))
require.Equal(t, c.wantMeta, listResp.Meta)

Expand Down
Loading
Loading