From 5873498936263dc051264c0bbe31d8425b23ab43 Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 22 Nov 2019 11:26:40 +0100 Subject: [PATCH 01/12] Fix fetching locations from PIs --- backend/audits/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/audits/tasks.py b/backend/audits/tasks.py index bb8523aa8..9651a86b6 100644 --- a/backend/audits/tasks.py +++ b/backend/audits/tasks.py @@ -281,7 +281,9 @@ def get_wpt_audit_configurations(): for location, location_data in data.items(): browsers = location_data["Browsers"].split(",") - group = location_data["group"] + group = location_data.get( + "group", "" + ) # Private instances locations may not be grouped label = location_data["labelShort"] for brower in browsers: configuration, created = AvailableAuditParameters.objects.update_or_create( From fb8aea3891183fc6993c9c2307830241e856e8b6 Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 22 Nov 2019 12:02:58 +0100 Subject: [PATCH 02/12] Add wpt_instance_url field --- .../migrations/0031_auto_20191122_1154.py | 21 +++++++++++++++++++ backend/projects/models.py | 6 ++++++ backend/projects/serializers.py | 9 +++++++- 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 backend/projects/migrations/0031_auto_20191122_1154.py diff --git a/backend/projects/migrations/0031_auto_20191122_1154.py b/backend/projects/migrations/0031_auto_20191122_1154.py new file mode 100644 index 000000000..0746441b3 --- /dev/null +++ b/backend/projects/migrations/0031_auto_20191122_1154.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.4 on 2019-11-22 10:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("projects", "0030_auto_20190904_1152")] + + operations = [ + migrations.AddField( + model_name="availableauditparameters", + name="wpt_instance_url", + field=models.CharField(default="https://webpagetest.org", max_length=100), + ), + migrations.AddField( + model_name="project", + name="wpt_instance_url", + field=models.CharField(default="https://webpagetest.org", max_length=100), + ), + ] diff --git a/backend/projects/models.py b/backend/projects/models.py index 7baf086bd..282a65ef1 100644 --- a/backend/projects/models.py +++ b/backend/projects/models.py @@ -14,6 +14,9 @@ class Project(BaseModel): User, blank=True, related_name="member_of", through="ProjectMemberRole" ) is_active = models.BooleanField(default=True) + wpt_instance_url = models.CharField( + max_length=100, blank=False, null=False, default="https://webpagetest.org" + ) @property def latest_audit_at(self): @@ -108,6 +111,9 @@ class AvailableAuditParameters(BaseModel): location_label = models.CharField(max_length=100, blank=False, null=False) location_group = models.CharField(max_length=100, blank=False, null=False) is_active = models.BooleanField(default=True) + wpt_instance_url = models.CharField( + max_length=100, blank=False, null=False, default="https://webpagetest.org" + ) class Meta: ordering = ("location", "browser") diff --git a/backend/projects/serializers.py b/backend/projects/serializers.py index b07e85dcb..176d8043d 100644 --- a/backend/projects/serializers.py +++ b/backend/projects/serializers.py @@ -86,7 +86,13 @@ class Meta: class AvailableAuditParameterSerializer(serializers.ModelSerializer): class Meta: model = AvailableAuditParameters - fields = ("uuid", "browser", "location_label", "location_group") + fields = ( + "uuid", + "browser", + "location_label", + "location_group", + "wpt_instance_url", + ) class ProjectAuditParametersSerializer(serializers.ModelSerializer): @@ -156,5 +162,6 @@ class Meta: "screenshot_url", "latest_audit_at", "wpt_api_key", + "wpt_instance_url", "has_siblings", ) From 4e250348c9226b1febbb7d04630b5a7945e92dea Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 22 Nov 2019 16:13:27 +0100 Subject: [PATCH 03/12] Filter available audit parameters via WPT instance URL --- .../EnvironmentSettings.tsx | 19 +++++++++++++++---- .../src/redux/entities/projects/modelizer.ts | 3 ++- frontend/src/redux/entities/projects/types.ts | 2 ++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/frontend/src/pages/EnvironmentSettings/EnvironmentSettings.tsx b/frontend/src/pages/EnvironmentSettings/EnvironmentSettings.tsx index fc391b30b..61354158a 100644 --- a/frontend/src/pages/EnvironmentSettings/EnvironmentSettings.tsx +++ b/frontend/src/pages/EnvironmentSettings/EnvironmentSettings.tsx @@ -47,16 +47,18 @@ const EnvironmentSettings: React.FunctionComponent = ({ browser: string, location_label: string, location_group: string, + wpt_instance_url: string, } useFetchProjectIfUndefined(fetchProjectsRequest, match.params.projectId, project); - const [availableAuditParameters, setAvailableAuditParameters] = React.useState>([]) + const [availableAuditParameters, setAvailableAuditParameters] = React.useState>([]) const modelizeAvailableAuditParameters = (apiAvailableAuditParameters: ApiAvailableAuditParameters) => ({ label: `${apiAvailableAuditParameters.location_label}. ${apiAvailableAuditParameters.browser}`, uuid: apiAvailableAuditParameters.uuid, + wptInstanceURL: apiAvailableAuditParameters.wpt_instance_url }); React.useEffect( @@ -65,7 +67,12 @@ const EnvironmentSettings: React.FunctionComponent = ({ request .then((response) => { if(response) { - setAvailableAuditParameters(response.body.map((apiAvailableAuditParameters: ApiAvailableAuditParameters) => modelizeAvailableAuditParameters(apiAvailableAuditParameters))); + setAvailableAuditParameters( + response.body + .map((apiAvailableAuditParameters: ApiAvailableAuditParameters) => + modelizeAvailableAuditParameters(apiAvailableAuditParameters) + ) + ); } }) }, @@ -129,6 +136,10 @@ const EnvironmentSettings: React.FunctionComponent = ({ ); } + const filteredAvailableProjectsParameters = availableAuditParameters.filter( + availableAuditParameter => availableAuditParameter.wptInstanceURL === project.wptInstanceURL + ) + return ( {intl.formatMessage({ id: 'ProjectSettings.settings'}) + ' - ' + project.name} @@ -149,13 +160,13 @@ const EnvironmentSettings: React.FunctionComponent = ({ disabled={!isUserAdminOfProject(currentUser, project)} projectId={project.uuid} auditParameterId={auditParameterId} - availableAuditParameters={availableAuditParameters} + availableAuditParameters={filteredAvailableProjectsParameters} /> ))} {isUserAdminOfProject(currentUser, project) && } diff --git a/frontend/src/redux/entities/projects/modelizer.ts b/frontend/src/redux/entities/projects/modelizer.ts index 14f6f7b1d..acff672a6 100644 --- a/frontend/src/redux/entities/projects/modelizer.ts +++ b/frontend/src/redux/entities/projects/modelizer.ts @@ -10,7 +10,8 @@ export const modelizeProject = (project: ApiProjectType): Record modelizeProjectMember(apiProjectMember)), - wptApiKey: project.wpt_api_key + wptApiKey: project.wpt_api_key, + wptInstanceURL: project.wpt_instance_url }, }); diff --git a/frontend/src/redux/entities/projects/types.ts b/frontend/src/redux/entities/projects/types.ts index 19b5cd407..b7a218609 100644 --- a/frontend/src/redux/entities/projects/types.ts +++ b/frontend/src/redux/entities/projects/types.ts @@ -13,6 +13,7 @@ export interface ProjectType { auditParametersIds: string[]; projectMembers: ProjectMember[]; wptApiKey: string; + wptInstanceURL: string; }; export interface ApiProjectType { @@ -25,6 +26,7 @@ export interface ApiProjectType { latest_audit_at: string; project_members: ApiProjectMember[]; wpt_api_key: string; + wpt_instance_url: string; has_siblings: boolean; }; From 5931a5f818e31c47082860b88f7237ea0405e734 Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 22 Nov 2019 17:03:32 +0100 Subject: [PATCH 04/12] Fix typo --- backend/audits/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/audits/tasks.py b/backend/audits/tasks.py index 9651a86b6..a6a7bb81a 100644 --- a/backend/audits/tasks.py +++ b/backend/audits/tasks.py @@ -285,9 +285,9 @@ def get_wpt_audit_configurations(): "group", "" ) # Private instances locations may not be grouped label = location_data["labelShort"] - for brower in browsers: + for browser in browsers: configuration, created = AvailableAuditParameters.objects.update_or_create( - browser=brower, + browser=browser, location=location, defaults={ "location_label": label, From 90f0e98eb382cfb92c1213fd81b7c1b6d8c28d44 Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 22 Nov 2019 17:40:21 +0100 Subject: [PATCH 05/12] Create POST /projects/available_audit_parameters/discover endpoint --- backend/audits/tasks.py | 24 ++++++++++++++++----- backend/projects/urls.py | 3 +++ backend/projects/views.py | 44 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/backend/audits/tasks.py b/backend/audits/tasks.py index a6a7bb81a..e4b5c1286 100644 --- a/backend/audits/tasks.py +++ b/backend/audits/tasks.py @@ -259,13 +259,22 @@ def clean_old_audit_statuses(): @shared_task -def get_wpt_audit_configurations(): +def get_wpt_audit_configurations(wpt_instance_url="https://webpagetest.org"): """gets all the available locations from WPT""" - response = requests.get("https://www.webpagetest.org/getLocations.php?f=json&k=A") + + # For some reason, the key mask to get API-available locations is different between + # public and private WPT instances + wpt_key_mask = "" + if wpt_instance_url == "https://webpagetest.org": + wpt_key_mask = "A" + + response = requests.get( + f"{wpt_instance_url}/getLocations.php?f=json&k={wpt_key_mask}" + ) if response.status_code != 200: logging.error("Invalid response from WebPageTest API: non-200 response code") - return + raise Exception("Invalid response from WebPageTest API: non-200 response code") try: data = response.json()["data"] @@ -273,9 +282,13 @@ def get_wpt_audit_configurations(): logging.error( "Invalid response from WebPageTest API: 'data' key is not present" ) - return + raise Exception( + "Invalid response from WebPageTest API: 'data' key is not present" + ) - for available_audit_parameter in AvailableAuditParameters.objects.all(): + for available_audit_parameter in AvailableAuditParameters.objects.filter( + wpt_instance_url=wpt_instance_url + ): available_audit_parameter.is_active = False available_audit_parameter.save() @@ -294,4 +307,5 @@ def get_wpt_audit_configurations(): "location_group": group, "is_active": True, }, + wpt_instance_url=wpt_instance_url, ) diff --git a/backend/projects/urls.py b/backend/projects/urls.py index c53f2ad19..d261fe744 100644 --- a/backend/projects/urls.py +++ b/backend/projects/urls.py @@ -17,6 +17,9 @@ views.project_audit_parameters_detail, ), path("available_audit_parameters", views.available_audit_parameters), + path( + "available_audit_parameters/discover", views.discover_available_audit_parameters + ), path("/scripts", views.project_scripts), path("/scripts/", views.project_script_detail), ] diff --git a/backend/projects/views.py b/backend/projects/views.py index d655d86eb..cf5edccdd 100644 --- a/backend/projects/views.py +++ b/backend/projects/views.py @@ -6,6 +6,7 @@ from rest_framework import permissions, status from rest_framework.decorators import api_view, permission_classes from rest_framework.parsers import JSONParser +from requests.exceptions import ConnectionError from projects.models import ( Page, Project, @@ -28,6 +29,8 @@ is_admin_of_project, ) +from audits.tasks import get_wpt_audit_configurations + def get_user_projects(user_id): return Project.objects.filter(members__id=user_id, is_active=True) @@ -404,6 +407,47 @@ def project_members(request, project_uuid): ) +@swagger_auto_schema( + methods=["post"], + request_body="", + responses={ + 201: openapi.Response( + "Returns discovered available audit parameters for the WPT instance URL passed in parameter", + AvailableAuditParameterSerializer, + ) + }, + tags=["Project Audit Parameters"], +) +@api_view(["POST"]) +def discover_available_audit_parameters(request): + data = JSONParser().parse(request) + if "wpt_instance_url" in data: + try: + get_wpt_audit_configurations(data["wpt_instance_url"]) + except ConnectionError: + return JsonResponse( + { + "error": "UNREACHABLE", + "details": "The WPT instance is not reachable, please check the URL", + }, + status=status.HTTP_400_BAD_REQUEST, + ) + available_audit_parameters = AvailableAuditParameters.objects.filter( + is_active=True + ) + serializer = AvailableAuditParameterSerializer( + available_audit_parameters, many=True + ) + return JsonResponse(serializer.data, safe=False) + return JsonResponse( + { + "error": "MISSING_PARAMETER", + "details": "You must provide a wpt_instance_url in the request body", + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + @swagger_auto_schema( methods=["get"], responses={ From cac2078acdc333ae6fdc423e6224783655c3b04c Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 22 Nov 2019 17:42:37 +0100 Subject: [PATCH 06/12] Update Swagger docs --- backend/projects/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/projects/views.py b/backend/projects/views.py index cf5edccdd..2ef38c268 100644 --- a/backend/projects/views.py +++ b/backend/projects/views.py @@ -409,7 +409,9 @@ def project_members(request, project_uuid): @swagger_auto_schema( methods=["post"], - request_body="", + request_body=openapi.Schema( + type="object", properties={"wpt_instance_url": openapi.Schema(type="string")} + ), responses={ 201: openapi.Response( "Returns discovered available audit parameters for the WPT instance URL passed in parameter", From 2f9784adc89e93360cd7f7cfdf7c8ce7b7a8ca05 Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 22 Nov 2019 18:31:30 +0100 Subject: [PATCH 07/12] Remove hardcoded mentions of webpagetest.org --- backend/audits/tasks.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/audits/tasks.py b/backend/audits/tasks.py index e4b5c1286..cbb8e6d36 100644 --- a/backend/audits/tasks.py +++ b/backend/audits/tasks.py @@ -37,11 +37,13 @@ def request_audit(audit_uuid): payload["url"] = audit.page.url payload["lighthouse"] = 1 payload["k"] = audit.page.project.wpt_api_key + wpt_instance_url = audit.page.project.wpt_instance_url elif audit.script is not None: payload["script"] = audit.script.script payload["k"] = audit.script.project.wpt_api_key + wpt_instance_url = audit.script.project.wpt_instance_url - r = requests.post("https://www.webpagetest.org/runtest.php", params=payload) + r = requests.post(f"{wpt_instance_url}/runtest.php", params=payload) response = r.json() if response["statusCode"] == 200: audit_status_queueing = AuditStatusHistory( @@ -85,9 +87,14 @@ def poll_audit_results(audit_uuid, json_url): audit_status_requested.save() poll_audit_results.apply_async((audit_uuid, json_url), countdown=15) elif status_code == 200: + if audit.page is not None: + wpt_instance_url = audit.page.project.wpt_instance_url + elif audit.script is not None: + wpt_instance_url = audit.script.project.wpt_instance_url + parsed_url = urlparse(json_url) test_id = parse_qs(parsed_url.query)["test"][0] - wpt_results_user_url = f"https://www.webpagetest.org/result/{test_id}" + wpt_results_user_url = f"{wpt_instance_url}/result/{test_id}" try: if audit.page is not None: project = audit.page.project From 37a7cf082b3d20f3cbec36bafaaa9223c5ca7ccc Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 22 Nov 2019 18:32:45 +0100 Subject: [PATCH 08/12] Allow admins to change WPT Instance URL --- .../GeneralSettings/GeneralSettings.style.tsx | 9 + .../pages/GeneralSettings/GeneralSettings.tsx | 91 ++-- .../GeneralSettings/GeneralSettings.wrap.tsx | 19 +- .../src/redux/entities/projects/actions.ts | 2 +- frontend/src/redux/entities/projects/sagas.ts | 401 +++++++++++------- frontend/src/translations/en.json | 2 + frontend/src/translations/fr.json | 2 + 7 files changed, 325 insertions(+), 201 deletions(-) diff --git a/frontend/src/pages/GeneralSettings/GeneralSettings.style.tsx b/frontend/src/pages/GeneralSettings/GeneralSettings.style.tsx index b1765d5f7..2896b0cd8 100644 --- a/frontend/src/pages/GeneralSettings/GeneralSettings.style.tsx +++ b/frontend/src/pages/GeneralSettings/GeneralSettings.style.tsx @@ -142,6 +142,15 @@ MemberAdminDeleteButton: styled.button` SettingsFieldContainer: styled.div` margin-top: ${getSpacing(4)}; `, + +ExplanationText: styled.span` + margin-top: ${getSpacing(4)}; + line-height: ${lineHeight.bodyText}; + color: ${colorUsage.bodyText}; + font-family: ${fontFamily.mainSans}; + font-size: ${fontSize.bodyText}; + font-weight: ${fontWeight.bodyText}; +`, }; export default Style; diff --git a/frontend/src/pages/GeneralSettings/GeneralSettings.tsx b/frontend/src/pages/GeneralSettings/GeneralSettings.tsx index 2ec02493e..16824ab37 100644 --- a/frontend/src/pages/GeneralSettings/GeneralSettings.tsx +++ b/frontend/src/pages/GeneralSettings/GeneralSettings.tsx @@ -16,12 +16,15 @@ export type OwnProps = {} & RouteComponentProps<{ }>; type Props = { - currentUser: UserState, + currentUser: UserState; fetchProjectsRequest: (projectId: string) => void; project?: ProjectType | null; toastrDisplay: ProjectToastrDisplayType; setProjectToastrDisplay: (toastrDisplay: ProjectToastrDisplayType) => void; - editProjectDetailsRequest: (projectId: string, payload: {name: string, wpt_api_key: string}) => void; + editProjectDetailsRequest: ( + projectId: string, + payload: { name: string; wpt_api_key: string; wpt_instance_url: string }, + ) => void; } & OwnProps & InjectedIntlProps; @@ -35,48 +38,48 @@ const GeneralSettings: React.FunctionComponent = ({ setProjectToastrDisplay, editProjectDetailsRequest, }) => { - interface UserOption { value: string; label: string; disabled: boolean; - }; + } interface ApiAvailableAuditParameters { - uuid: string, - browser: string, - location_label: string, - location_group: string, + uuid: string; + browser: string; + location_label: string; + location_group: string; } - useFetchProjectIfUndefined(fetchProjectsRequest, match.params.projectId, project); const [projectName, setProjectName] = React.useState(''); const [projectApiKey, setProjectApiKey] = React.useState(''); + const [projectInstanceURL, setProjectInstanceURL] = React.useState(''); React.useEffect( () => { setProjectName(project ? project.name : ''); setProjectApiKey(project ? project.wptApiKey : ''); + setProjectInstanceURL(project ? project.wptInstanceURL : ''); }, - [project] - ) + [project], + ); React.useEffect( () => { - if('' !== toastrDisplay) { - switch(toastrDisplay) { - case "editProjectDetailsSuccess": - toastr.success( - intl.formatMessage({'id': 'Toastr.ProjectSettings.success_title'}), - intl.formatMessage({'id': 'Toastr.ProjectSettings.edit_project_details_success'}), - ); - break; - case "editProjectDetailsError": + if ('' !== toastrDisplay) { + switch (toastrDisplay) { + case 'editProjectDetailsSuccess': + toastr.success( + intl.formatMessage({ id: 'Toastr.ProjectSettings.success_title' }), + intl.formatMessage({ id: 'Toastr.ProjectSettings.edit_project_details_success' }), + ); + break; + case 'editProjectDetailsError': toastr.error( - intl.formatMessage({'id': 'Toastr.ProjectSettings.error_title'}), - intl.formatMessage({'id': 'Toastr.ProjectSettings.error_message'}), + intl.formatMessage({ id: 'Toastr.ProjectSettings.error_title' }), + intl.formatMessage({ id: 'Toastr.ProjectSettings.error_message' }), ); break; } @@ -105,30 +108,33 @@ const GeneralSettings: React.FunctionComponent = ({ ); } - const handleNameChange = (e: React.SyntheticEvent) => { - setProjectName(e.currentTarget.value) - } + setProjectName(e.currentTarget.value); + }; const handleApiKeyChange = (e: React.SyntheticEvent) => { - setProjectApiKey(e.currentTarget.value) - } + setProjectApiKey(e.currentTarget.value); + }; + + const handleInstanceURLChange = (e: React.SyntheticEvent) => { + setProjectInstanceURL(e.currentTarget.value); + }; const sendEditRequestOnBlur = () => { - editProjectDetailsRequest( - project.uuid, - { - name: projectName, - wpt_api_key: projectApiKey, - }, - ) + editProjectDetailsRequest(project.uuid, { + name: projectName, + wpt_api_key: projectApiKey, + wpt_instance_url: projectInstanceURL, + }); }; return ( - {intl.formatMessage({ id: 'ProjectSettings.settings'}) + ' - ' + project.name} + + {intl.formatMessage({ id: 'ProjectSettings.settings' }) + ' - ' + project.name} + - + = ({ value={projectApiKey} /> + + + + + {intl.formatMessage({id: "ProjectSettings.wpt_instance_url_explanation"})} + = ({ /> ); -} +}; export default GeneralSettings; diff --git a/frontend/src/pages/GeneralSettings/GeneralSettings.wrap.tsx b/frontend/src/pages/GeneralSettings/GeneralSettings.wrap.tsx index 1b0578456..730872f31 100644 --- a/frontend/src/pages/GeneralSettings/GeneralSettings.wrap.tsx +++ b/frontend/src/pages/GeneralSettings/GeneralSettings.wrap.tsx @@ -3,7 +3,11 @@ import { connect } from 'react-redux'; import { Dispatch } from 'redux'; import { RootState } from 'redux/types'; -import { editProjectDetailsRequest, fetchProjectsRequest, setProjectToastrDisplay } from 'redux/entities/projects'; +import { + editProjectDetailsRequest, + fetchProjectsRequest, + setProjectToastrDisplay, +} from 'redux/entities/projects'; import { getProject, getProjectToastrDisplay } from 'redux/entities/projects/selectors'; import { ProjectToastrDisplayType } from 'redux/entities/projects/types'; import { getUser } from 'redux/user/selectors'; @@ -12,13 +16,18 @@ import GeneralSettings, { OwnProps } from './GeneralSettings'; const mapStateToProps = (state: RootState, props: OwnProps) => ({ currentUser: getUser(state), project: getProject(state, props.match.params.projectId), - toastrDisplay: getProjectToastrDisplay(state) + toastrDisplay: getProjectToastrDisplay(state), }); const mapDispatchToProps = (dispatch: Dispatch) => ({ - fetchProjectsRequest: (projectId: string) => dispatch(fetchProjectsRequest({ currentProjectId: projectId })), - setProjectToastrDisplay: (toastrDisplay: ProjectToastrDisplayType) => dispatch(setProjectToastrDisplay({ toastrDisplay })), - editProjectDetailsRequest: (projectId: string, payload: {name: string, wpt_api_key: string}) => dispatch(editProjectDetailsRequest({ projectId, payload })), + fetchProjectsRequest: (projectId: string) => + dispatch(fetchProjectsRequest({ currentProjectId: projectId })), + setProjectToastrDisplay: (toastrDisplay: ProjectToastrDisplayType) => + dispatch(setProjectToastrDisplay({ toastrDisplay })), + editProjectDetailsRequest: ( + projectId: string, + payload: { name: string; wpt_api_key: string; wpt_instance_url: string }, + ) => dispatch(editProjectDetailsRequest({ projectId, payload })), }); export default connect( diff --git a/frontend/src/redux/entities/projects/actions.ts b/frontend/src/redux/entities/projects/actions.ts index 7740c4d88..e239484e4 100644 --- a/frontend/src/redux/entities/projects/actions.ts +++ b/frontend/src/redux/entities/projects/actions.ts @@ -90,7 +90,7 @@ export const setProjectToastrDisplay = createStandardAction('projects/SET_TOASTR }>(); export const editProjectDetailsRequest = createStandardAction('projects/EDIT_PROJECT_DETAILS_REQUEST')<{ projectId: string; - payload: { name: string, wpt_api_key: string }; + payload: { name: string, wpt_api_key: string, wpt_instance_url: string }; }>(); export const editProjectDetailsError = createStandardAction('projects/EDIT_PROJECT_DETAILS_ERROR')<{ projectId: string | null; diff --git a/frontend/src/redux/entities/projects/sagas.ts b/frontend/src/redux/entities/projects/sagas.ts index 7951462d7..d50fd913c 100644 --- a/frontend/src/redux/entities/projects/sagas.ts +++ b/frontend/src/redux/entities/projects/sagas.ts @@ -1,10 +1,18 @@ import { all, call, put, takeEvery } from 'redux-saga/effects'; import { handleAPIExceptions } from 'services/networking/handleAPIExceptions'; -import { makeDeleteRequest, makeGetRequest, makePostRequest, makePutRequest } from 'services/networking/request'; +import { + makeDeleteRequest, + makeGetRequest, + makePostRequest, + makePutRequest, +} from 'services/networking/request'; import { ActionType, getType } from 'typesafe-actions'; import { fetchAuditParametersAction } from '../auditParameters/actions'; -import { modelizeApiAuditParametersListToById, modelizeAuditParameters } from '../auditParameters/modelizer'; +import { + modelizeApiAuditParametersListToById, + modelizeAuditParameters, +} from '../auditParameters/modelizer'; import { ApiAuditParametersType } from '../auditParameters/types'; import { pollAuditStatusAction } from '../audits'; import { fetchAuditStatusHistoriesAction } from '../auditStatusHistories'; @@ -53,55 +61,69 @@ import { modelizeProject, modelizeProjects } from './modelizer'; import { ApiProjectType } from './types'; function* fetchProjectsFailedHandler(error: Error, actionPayload: Record) { - yield put(fetchProjectError({ projectId: actionPayload.currentProjectId, errorMessage: error.message })); -}; + yield put( + fetchProjectError({ projectId: actionPayload.currentProjectId, errorMessage: error.message }), + ); +} function* fetchProjectFailedHandler(error: Error, actionPayload: Record) { yield put(fetchProjectError({ projectId: actionPayload.projectId, errorMessage: error.message })); -}; +} function* addMemberToProjectFailedHandler(error: Error, actionPayload: Record) { - yield put(addMemberToProjectError({ projectId: actionPayload.projectId, errorMessage: error.message })); + yield put( + addMemberToProjectError({ projectId: actionPayload.projectId, errorMessage: error.message }), + ); yield put(setProjectToastrDisplay({ toastrDisplay: 'addMemberError' })); -}; +} function* addPageToProjectFailedHandler(error: Error, actionPayload: Record) { - yield put(addPageToProjectError({ projectId: actionPayload.projectId, errorMessage: error.message })); + yield put( + addPageToProjectError({ projectId: actionPayload.projectId, errorMessage: error.message }), + ); yield put(setProjectToastrDisplay({ toastrDisplay: 'addPageError' })); -}; +} function* deleteMemberOfProjectFailedHandler(error: Error, actionPayload: Record) { - yield put(deleteMemberOfProjectError({ - projectId: actionPayload.projectId, - userId: actionPayload.userId, - errorMessage: error.message - })); -}; + yield put( + deleteMemberOfProjectError({ + projectId: actionPayload.projectId, + userId: actionPayload.userId, + errorMessage: error.message, + }), + ); +} function* deletePageOfProjectFailedHandler(error: Error, actionPayload: Record) { - yield put(deletePageOfProjectError({ - projectId: actionPayload.projectId, - pageId: actionPayload.pageId, - errorMessage: error.message - })); + yield put( + deletePageOfProjectError({ + projectId: actionPayload.projectId, + pageId: actionPayload.pageId, + errorMessage: error.message, + }), + ); yield put(setProjectToastrDisplay({ toastrDisplay: 'deletePageError' })); -}; +} function* editMemberOfProjectFailedHandler(error: Error, actionPayload: Record) { - yield put(editMemberOfProjectError({ - projectId: actionPayload.projectId, - userId: actionPayload.userId, - errorMessage: error.message - })); -}; + yield put( + editMemberOfProjectError({ + projectId: actionPayload.projectId, + userId: actionPayload.userId, + errorMessage: error.message, + }), + ); +} function* editPageFailedHandler(error: Error, actionPayload: Record) { - yield put(editPageError({ - projectId: actionPayload.projectId, - page: actionPayload.page - })); + yield put( + editPageError({ + projectId: actionPayload.projectId, + page: actionPayload.page, + }), + ); yield put(setProjectToastrDisplay({ toastrDisplay: 'editPageError' })); -}; +} function* fetchProjects(action: ActionType) { // this sagas will use a sort of lazy loading to start loading one project first, in order to speed @@ -120,13 +142,13 @@ function* fetchProjects(action: ActionType) { if (firstProject.uuid) { yield put(saveFetchedProjects({ projects: [firstProject] })); } else { - yield put(fetchProjectError({ projectId: null, errorMessage: "No project returned" })); + yield put(fetchProjectError({ projectId: null, errorMessage: 'No project returned' })); return; } // if the user has no other project, do not fetch them if (!firstProject.has_siblings) { return; - }; + } const endpoint = '/api/projects/'; const { body: projects }: { body: ApiProjectType[] } = yield call( @@ -136,7 +158,7 @@ function* fetchProjects(action: ActionType) { null, ); yield put(saveFetchedProjects({ projects })); -}; +} function* fetchProject(action: ActionType) { const endpoint = `/api/projects/${action.payload.projectId}/`; @@ -147,7 +169,7 @@ function* fetchProject(action: ActionType) { null, ); yield put(saveFetchedProjects({ projects: [project] })); -}; +} function* addMemberToProject(action: ActionType) { yield put(setProjectToastrDisplay({ toastrDisplay: '' })); @@ -160,7 +182,7 @@ function* addMemberToProject(action: ActionType) { yield put(setProjectToastrDisplay({ toastrDisplay: '' })); @@ -171,105 +193,145 @@ function* addPageToProject(action: ActionType) { true, { name: action.payload.pageName, url: action.payload.pageUrl }, ); - yield put(addPageToProjectSuccess({ projectId: action.payload.projectId, page: modelizePage(pageResponse) })); - yield put(fetchPageAction.success({ - byId: { - [pageResponse.uuid]: modelizePage(pageResponse) - }, - })); + yield put( + addPageToProjectSuccess({ + projectId: action.payload.projectId, + page: modelizePage(pageResponse), + }), + ); + yield put( + fetchPageAction.success({ + byId: { + [pageResponse.uuid]: modelizePage(pageResponse), + }, + }), + ); yield put(setProjectToastrDisplay({ toastrDisplay: 'addPageSuccess' })); -}; +} function* deleteMemberOfProject(action: ActionType) { const endpoint = `/api/projects/${action.payload.projectId}/members/${action.payload.userId}`; - yield call( - makeDeleteRequest, - endpoint, - true, + yield call(makeDeleteRequest, endpoint, true); + yield put( + deleteMemberOfProjectSuccess({ + projectId: action.payload.projectId, + userId: action.payload.userId, + }), ); - yield put(deleteMemberOfProjectSuccess({ projectId: action.payload.projectId, userId: action.payload.userId })); -}; +} function* deletePageOfProject(action: ActionType) { const endpoint = `/api/projects/${action.payload.projectId}/pages/${action.payload.pageId}`; - yield call( - makeDeleteRequest, - endpoint, - true, + yield call(makeDeleteRequest, endpoint, true); + yield put( + deletePageOfProjectSuccess({ + projectId: action.payload.projectId, + pageId: action.payload.pageId, + }), ); - yield put(deletePageOfProjectSuccess({ projectId: action.payload.projectId, pageId: action.payload.pageId })); yield put(setProjectToastrDisplay({ toastrDisplay: 'deletePageSuccess' })); -}; +} function* editMemberOfProject(action: ActionType) { const endpoint = `/api/projects/${action.payload.projectId}/members/${action.payload.userId}`; - yield call( - makePutRequest, - endpoint, - true, - { is_admin: action.payload.isAdmin } - ); - yield put(editMemberOfProjectSuccess({ - projectId: action.payload.projectId, - userId: action.payload.userId, - isAdmin: action.payload.isAdmin - })); -}; + yield call(makePutRequest, endpoint, true, { is_admin: action.payload.isAdmin }); + yield put( + editMemberOfProjectSuccess({ + projectId: action.payload.projectId, + userId: action.payload.userId, + isAdmin: action.payload.isAdmin, + }), + ); +} function* editPage(action: ActionType) { const endpoint = `/api/projects/${action.payload.projectId}/pages/${action.payload.page.uuid}`; - const { body: pageResponse }: { body: ApiPageType } = yield call( - makePutRequest, - endpoint, - true, - { name: action.payload.page.name, url: action.payload.page.url } - ); - yield put(editPageSuccess({page: modelizePage(pageResponse)})); + const { body: pageResponse }: { body: ApiPageType } = yield call(makePutRequest, endpoint, true, { + name: action.payload.page.name, + url: action.payload.page.url, + }); + yield put(editPageSuccess({ page: modelizePage(pageResponse) })); yield put(setProjectToastrDisplay({ toastrDisplay: 'editPageSuccess' })); -}; +} function* saveProjectsToStore(action: ActionType) { const projects = action.payload.projects; - yield put(fetchPageAction.success({ - byId: modelizeApiPagesToById(projects.reduce((apiPages: ApiPageType[], project: ApiProjectType) => { - return apiPages.concat(project.pages); - }, [])), - })); - yield put(fetchScriptAction.success({ - byId: modelizeApiScriptsToById(projects.reduce((apiScripts: ApiScriptType[], project: ApiProjectType) => { - return apiScripts.concat(project.scripts); - }, [])), - })); - yield put(fetchAuditParametersAction.success({ - byId: modelizeApiAuditParametersListToById(projects.reduce((apiAuditParametersList: ApiAuditParametersType[], project: ApiProjectType) => { - return apiAuditParametersList.concat(project.audit_parameters_list); - }, [])), - })); + yield put( + fetchPageAction.success({ + byId: modelizeApiPagesToById( + projects.reduce((apiPages: ApiPageType[], project: ApiProjectType) => { + return apiPages.concat(project.pages); + }, []), + ), + }), + ); + yield put( + fetchScriptAction.success({ + byId: modelizeApiScriptsToById( + projects.reduce((apiScripts: ApiScriptType[], project: ApiProjectType) => { + return apiScripts.concat(project.scripts); + }, []), + ), + }), + ); + yield put( + fetchAuditParametersAction.success({ + byId: modelizeApiAuditParametersListToById( + projects.reduce( + (apiAuditParametersList: ApiAuditParametersType[], project: ApiProjectType) => { + return apiAuditParametersList.concat(project.audit_parameters_list); + }, + [], + ), + ), + }), + ); - const allApiAuditStatusHistories = projects.reduce((apiAuditStatusHistories: ApiAuditStatusHistoryType[], project: ApiProjectType) => { - return apiAuditStatusHistories - .concat(project.pages.reduce((pageStatusHistories: ApiAuditStatusHistoryType[], page: ApiPageType) => { - return pageStatusHistories.concat(page.latest_audit_status_histories); - }, [])) - .concat(project.scripts.reduce((scriptStatusHistories: ApiAuditStatusHistoryType[], script: ApiScriptType) => { - return scriptStatusHistories.concat(script.latest_audit_status_histories); - }, [])); - }, []); - yield put(fetchAuditStatusHistoriesAction.success({ - byPageOrScriptIdAndAuditParametersId: modelizeApiAuditStatusHistoriesToByPageOrScriptIdAndAuditParametersId(allApiAuditStatusHistories), - })); + const allApiAuditStatusHistories = projects.reduce( + (apiAuditStatusHistories: ApiAuditStatusHistoryType[], project: ApiProjectType) => { + return apiAuditStatusHistories + .concat( + project.pages.reduce( + (pageStatusHistories: ApiAuditStatusHistoryType[], page: ApiPageType) => { + return pageStatusHistories.concat(page.latest_audit_status_histories); + }, + [], + ), + ) + .concat( + project.scripts.reduce( + (scriptStatusHistories: ApiAuditStatusHistoryType[], script: ApiScriptType) => { + return scriptStatusHistories.concat(script.latest_audit_status_histories); + }, + [], + ), + ); + }, + [], + ); + yield put( + fetchAuditStatusHistoriesAction.success({ + byPageOrScriptIdAndAuditParametersId: modelizeApiAuditStatusHistoriesToByPageOrScriptIdAndAuditParametersId( + allApiAuditStatusHistories, + ), + }), + ); // launch polling for all non-success and non-error auditStatusHistories - yield all(allApiAuditStatusHistories.map( - apiAuditStatusHistory => (apiAuditStatusHistory.status === "PENDING" || apiAuditStatusHistory.status === "REQUESTED") - ? put(pollAuditStatusAction({ - auditId: apiAuditStatusHistory.audit_id, - pageOrScriptId: apiAuditStatusHistory.page_id || apiAuditStatusHistory.script_id, - })) - // the all() effect requires effect types for all its children, so we use this useless call effect - : call(() => null) - )); + yield all( + allApiAuditStatusHistories.map(apiAuditStatusHistory => + apiAuditStatusHistory.status === 'PENDING' || apiAuditStatusHistory.status === 'REQUESTED' + ? put( + pollAuditStatusAction({ + auditId: apiAuditStatusHistory.audit_id, + pageOrScriptId: apiAuditStatusHistory.page_id || apiAuditStatusHistory.script_id, + }), + ) + : // the all() effect requires effect types for all its children, so we use this useless call effect + call(() => null), + ), + ); yield put(fetchProjectSuccess({ byId: modelizeProjects(projects) })); -}; +} function* editProjectDetails(action: ActionType) { const endpoint = `/api/projects/${action.payload.projectId}/`; @@ -277,19 +339,24 @@ function* editProjectDetails(action: ActionType) { - yield put(editProjectDetailsError({ - projectId: actionPayload.projectId, - errorMessage: error.message - })); + yield put( + editProjectDetailsError({ + projectId: actionPayload.projectId, + errorMessage: error.message, + }), + ); yield put(setProjectToastrDisplay({ toastrDisplay: 'editProjectDetailsError' })); -}; +} function* addAuditParameterToProject(action: ActionType) { const endpoint = `/api/projects/${action.payload.projectId}/audit_parameters`; @@ -303,42 +370,63 @@ function* addAuditParameterToProject(action: ActionType) { - yield put(addAuditParameterToProjectError({ projectId: actionPayload.projectId, errorMessage: error.message })); +function* addAuditParameterToProjectFailedHandler( + error: Error, + actionPayload: Record, +) { + yield put( + addAuditParameterToProjectError({ + projectId: actionPayload.projectId, + errorMessage: error.message, + }), + ); yield put(setProjectToastrDisplay({ toastrDisplay: 'addAuditParameterError' })); -}; +} -function* deleteAuditParameterFromProject(action: ActionType) { - const endpoint = `/api/projects/${action.payload.projectId}/audit_parameters/${action.payload.auditParameterId}`; - yield call( - makeDeleteRequest, - endpoint, - true, +function* deleteAuditParameterFromProject( + action: ActionType, +) { + const endpoint = `/api/projects/${action.payload.projectId}/audit_parameters/${ + action.payload.auditParameterId + }`; + yield call(makeDeleteRequest, endpoint, true); + yield put( + deleteAuditParameterFromProjectSuccess({ + projectId: action.payload.projectId, + auditParameterId: action.payload.auditParameterId, + }), ); - yield put(deleteAuditParameterFromProjectSuccess({ projectId: action.payload.projectId, auditParameterId: action.payload.auditParameterId })); yield put(setProjectToastrDisplay({ toastrDisplay: 'deleteAuditParameterSuccess' })); -}; +} -function* deleteAuditParameterFromProjectFailedHandler(error: Error, actionPayload: Record) { - yield put(deleteAuditParameterFromProjectError({ - projectId: actionPayload.projectId, - errorMessage: error.message - })); +function* deleteAuditParameterFromProjectFailedHandler( + error: Error, + actionPayload: Record, +) { + yield put( + deleteAuditParameterFromProjectError({ + projectId: actionPayload.projectId, + errorMessage: error.message, + }), + ); yield put(setProjectToastrDisplay({ toastrDisplay: 'deleteAuditParameterError' })); -}; - +} export default function* projectsSaga() { yield takeEvery( @@ -357,10 +445,7 @@ export default function* projectsSaga() { getType(addPageToProjectRequest), handleAPIExceptions(addPageToProject, addPageToProjectFailedHandler), ); - yield takeEvery( - getType(editPageRequest), - handleAPIExceptions(editPage, editPageFailedHandler), - ); + yield takeEvery(getType(editPageRequest), handleAPIExceptions(editPage, editPageFailedHandler)); yield takeEvery( getType(deleteMemberOfProjectRequest), handleAPIExceptions(deleteMemberOfProject, deleteMemberOfProjectFailedHandler), @@ -373,10 +458,7 @@ export default function* projectsSaga() { getType(fetchProjectsRequest), handleAPIExceptions(fetchProjects, fetchProjectsFailedHandler), ); - yield takeEvery( - getType(saveFetchedProjects), - saveProjectsToStore, - ); + yield takeEvery(getType(saveFetchedProjects), saveProjectsToStore); yield takeEvery( getType(editProjectDetailsRequest), handleAPIExceptions(editProjectDetails, editProjectDetailsFailedHandler), @@ -387,6 +469,9 @@ export default function* projectsSaga() { ); yield takeEvery( getType(deleteAuditParameterFromProjectRequest), - handleAPIExceptions(deleteAuditParameterFromProject, deleteAuditParameterFromProjectFailedHandler), + handleAPIExceptions( + deleteAuditParameterFromProject, + deleteAuditParameterFromProjectFailedHandler, + ), ); -}; +} diff --git a/frontend/src/translations/en.json b/frontend/src/translations/en.json index eb9079788..0ebc26a60 100644 --- a/frontend/src/translations/en.json +++ b/frontend/src/translations/en.json @@ -54,6 +54,8 @@ "general_settings": "General", "name": "Project name", "wpt_key": "WebPageTest API Key", + "wpt_instance_url": "WebPageTest instance URL", + "wpt_instance_url_explanation": "If you don’t use a private WebPageTest instance, leave the default value “https://webpagetest.org”. If you want to use a private instance, enter its URL without the trailing slash.", "project_audit_parameters": "Project environments", "pages_and_scripts": "Project pages and scripts", "scripts": "Project scripts", diff --git a/frontend/src/translations/fr.json b/frontend/src/translations/fr.json index 05e99da94..0c7571bbf 100644 --- a/frontend/src/translations/fr.json +++ b/frontend/src/translations/fr.json @@ -54,6 +54,8 @@ "general_settings": "Général", "name": "Nom du projet", "wpt_key": "Clé API WebPageTest", + "wpt_instance_url": "URL de l'instance WebPageTest", + "wpt_instance_url_explanation": "Si vous n'utilisez pas d'instance privée WebPageTest, laissez la valeur “https://webpagetest.org”. Si vous souhaitez utiliser une instance privée, saisissez l'URL de votre instance, sans caractère slash à la fin.", "project_audit_parameters": "Environnement du projet", "pages_and_scripts": "Pages et scripts du projet", "scripts": "Scripts du projet", From ffec6baa6e6526986671bb63709c93eed395c072 Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 22 Nov 2019 18:37:09 +0100 Subject: [PATCH 09/12] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a15c9e266..660d816ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Add support for WebPageTest Private Instances 🎉 (@phacks) ## [1.0.3] - 2019-12-10 From 5ce88464fff39a9b4868f11d0fe9f2529d6f2e9d Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Mon, 25 Nov 2019 12:20:52 +0100 Subject: [PATCH 10/12] Update instructions for switching instances --- frontend/src/translations/en.json | 2 +- frontend/src/translations/fr.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/translations/en.json b/frontend/src/translations/en.json index 0ebc26a60..ce08bbc15 100644 --- a/frontend/src/translations/en.json +++ b/frontend/src/translations/en.json @@ -55,7 +55,7 @@ "name": "Project name", "wpt_key": "WebPageTest API Key", "wpt_instance_url": "WebPageTest instance URL", - "wpt_instance_url_explanation": "If you don’t use a private WebPageTest instance, leave the default value “https://webpagetest.org”. If you want to use a private instance, enter its URL without the trailing slash.", + "wpt_instance_url_explanation": "If you don’t use a private WebPageTest instance, leave the default value “https://webpagetest.org”. If you want to use a private instance, enter its URL without the trailing slash. If you change the instance at some point in time, don’t forget to update the audit environments in the “Project environments” tab.", "project_audit_parameters": "Project environments", "pages_and_scripts": "Project pages and scripts", "scripts": "Project scripts", diff --git a/frontend/src/translations/fr.json b/frontend/src/translations/fr.json index 0c7571bbf..d44ce4f76 100644 --- a/frontend/src/translations/fr.json +++ b/frontend/src/translations/fr.json @@ -55,7 +55,7 @@ "name": "Nom du projet", "wpt_key": "Clé API WebPageTest", "wpt_instance_url": "URL de l'instance WebPageTest", - "wpt_instance_url_explanation": "Si vous n'utilisez pas d'instance privée WebPageTest, laissez la valeur “https://webpagetest.org”. Si vous souhaitez utiliser une instance privée, saisissez l'URL de votre instance, sans caractère slash à la fin.", + "wpt_instance_url_explanation": "Si vous n'utilisez pas d'instance privée WebPageTest, laissez la valeur “https://webpagetest.org”. Si vous souhaitez utiliser une instance privée, saisissez l'URL de votre instance, sans caractère slash à la fin. Si vous changez d'instance en cours de route, pensez à choisir à nouveau les environnements d'audit dans l'onglet “Environnements du projet”.", "project_audit_parameters": "Environnement du projet", "pages_and_scripts": "Pages et scripts du projet", "scripts": "Scripts du projet", From 27976e98cae4a94c5b5a6a211e186d993f8360aa Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Tue, 10 Dec 2019 18:15:27 +0100 Subject: [PATCH 11/12] Add tests to get_wpt_audit_configurations task --- .../json_mocks/wpt_GET_getLocations.json | 58 +++++++++++++++++++ backend/audits/tests/test_tasks.py | 49 +++++++++++++--- 2 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 backend/audits/tests/json_mocks/wpt_GET_getLocations.json diff --git a/backend/audits/tests/json_mocks/wpt_GET_getLocations.json b/backend/audits/tests/json_mocks/wpt_GET_getLocations.json new file mode 100644 index 000000000..2a701df6b --- /dev/null +++ b/backend/audits/tests/json_mocks/wpt_GET_getLocations.json @@ -0,0 +1,58 @@ +{ + "statusCode": 200, + "statusText": "Ok", + "data": { + "Dulles_MotoG4": { + "Label": "Moto G (gen 4)", + "location": "Dulles_MotoG4", + "Browsers": "Moto G4 - Chrome,Moto G4 - Firefox", + "status": "OK", + "relayServer": null, + "relayLocation": null, + "labelShort": "Dulles, VA", + "group": "Android Devices - Dulles, VA", + "PendingTests": { + "p1": 0, + "p2": 1, + "p3": 0, + "p4": 0, + "p5": 84, + "p6": 0, + "p7": 0, + "p8": 0, + "p9": 0, + "Total": 101, + "HighPriority": 0, + "LowPriority": 85, + "Testing": 16, + "Idle": 0 + } + }, + "Dulles_MotoG": { + "Label": "Moto G (gen 1)", + "location": "Dulles_MotoG", + "Browsers": "Moto G - Chrome", + "status": "OK", + "relayServer": null, + "relayLocation": null, + "labelShort": "Dulles, VA", + "group": "Android Devices - Dulles, VA", + "PendingTests": { + "p1": 0, + "p2": 0, + "p3": 0, + "p4": 0, + "p5": 0, + "p6": 0, + "p7": 0, + "p8": 0, + "p9": 0, + "Total": 0, + "HighPriority": 0, + "LowPriority": 0, + "Testing": 0, + "Idle": 13 + } + } + } +} diff --git a/backend/audits/tests/test_tasks.py b/backend/audits/tests/test_tasks.py index 066ebf5ed..97840909e 100644 --- a/backend/audits/tests/test_tasks.py +++ b/backend/audits/tests/test_tasks.py @@ -1,8 +1,7 @@ import httpretty from unittest.mock import MagicMock -import json -from django.test import TestCase, override_settings -from audits.tasks import request_audit +from django.test import TestCase +from audits.tasks import request_audit, get_wpt_audit_configurations from projects.models import ( Page, Project, @@ -15,22 +14,21 @@ class TasksTestCase(TestCase): @httpretty.activate - @override_settings(CELERY_EAGER=True) @patch("audits.tasks.poll_audit_results") def test_request_audit(self, poll_audit_results_mock): poll_audit_results_mock.apply_async = MagicMock() POST_runtest_data = open("audits/tests/json_mocks/wpt_POST_runtest.json").read() - GET_jsonResults_data = json.load( - open("audits/tests/json_mocks/wpt_GET_jsonResult.json") - ) + GET_jsonResults_data = open( + "audits/tests/json_mocks/wpt_GET_jsonResult.json" + ).read() httpretty.register_uri( httpretty.POST, - "https://www.webpagetest.org/runtest.php", + "https://webpagetest.org/runtest.php", body=POST_runtest_data, ) httpretty.register_uri( httpretty.GET, - "https://www.webpagetest.org/jsonResult.php?test=191024_HA_976b046886025ec8693cbe4f1145929e", + "https://webpagetest.org/jsonResult.php?test=191024_HA_976b046886025ec8693cbe4f1145929e", body=GET_jsonResults_data, ) project = Project.objects.create() @@ -71,3 +69,36 @@ def test_request_audit(self, poll_audit_results_mock): ), countdown=15, ) + + @httpretty.activate + def test_get_wpt_audit_configurations__create_configurations_for_default_instance( + self + ): + GET_getLocations_data = open( + "audits/tests/json_mocks/wpt_GET_getLocations.json" + ).read() + httpretty.register_uri( + httpretty.GET, + "https://webpagetest.org/getLocations.php?f=json&k=A", + body=GET_getLocations_data, + ) + get_wpt_audit_configurations() + available_audit_parameters = AvailableAuditParameters.objects.all() + self.assertEqual(len(available_audit_parameters), 3) + + @httpretty.activate + def test_get_wpt_audit_configurations__create_configurations_for_private_instance( + self + ): + GET_getLocations_data = open( + "audits/tests/json_mocks/wpt_GET_getLocations.json" + ).read() + private_instance_name = "http://myprivateinstance.com" + httpretty.register_uri( + httpretty.GET, + f"{private_instance_name}/getLocations.php?f=json&k=", + body=GET_getLocations_data, + ) + get_wpt_audit_configurations(private_instance_name) + available_audit_parameters = AvailableAuditParameters.objects.all() + self.assertEqual(len(available_audit_parameters), 3) From cf472fd99dd579be00b784e86609f390aa1fdb0e Mon Sep 17 00:00:00 2001 From: Nicolas Goutay Date: Fri, 13 Dec 2019 10:44:29 +0100 Subject: [PATCH 12/12] Add docs for Private Instances --- README.md | 3 +++ .../about-wpt-private-instances.md | 24 +++++++++++++++++++ .../setting-up-falco-for-private-instances.md | 20 ++++++++++++++++ docs/sidebars.js | 4 ++++ 4 files changed, 51 insertions(+) create mode 100644 docs/docs/wpt-private-instances/about-wpt-private-instances.md create mode 100644 docs/docs/wpt-private-instances/setting-up-falco-for-private-instances.md diff --git a/README.md b/README.md index e7807b6e2..f00d3d193 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ - 👥 Invite the whole team so that everyone (devs, ops, product, marketing…) is involved in performance - 🗺 Audit the performance of individual URLs or entire user journeys ([even on Single Page Apps!](https://css-tricks.com/recipes-for-performance-testing-single-page-applications-in-webpagetest/)) - 📸 Easily access and compare WebPageTest results between audits +- 🙈 Can be used with your own Private Instance of WebPageTest You can try a demo version by logging in to https://falco.theo.do with the credentials `demo / demodemo`. @@ -40,6 +41,8 @@ You can deploy Falco on Heroku by clicking on the following button: You will need to provide your credit card details to Heroku, but you will be under the free tier by default. You can find more details on why they are needed and Heroku’s pricing policy [in the docs](https://getfal.co). +After deployment, you can connect to Falco (and the admin interface at `/admin/`) with the credentials `admin` and `admin`: make sure to change your password after connecting! +
Heroku Teams user? Click here to deploy Falco.
diff --git a/docs/docs/wpt-private-instances/about-wpt-private-instances.md b/docs/docs/wpt-private-instances/about-wpt-private-instances.md new file mode 100644 index 000000000..eb64c1015 --- /dev/null +++ b/docs/docs/wpt-private-instances/about-wpt-private-instances.md @@ -0,0 +1,24 @@ +--- +id: about-wpt-private-instances +title: About WebPageTest Private Instances +sidebar_label: About WebPageTest Private Instances +--- + +WebPageTest is Open Source, meaning everyone can [access and modify its source code](https://github.com/WPO-Foundation/webpagetest)—meaning that it can be ran on one’s own infrastructure as well. + +Although the public instance, hosted at https://webpagetest.org, is perfectly suitable for most use cases, sometimes it can be useful to host one’s own Private Instance of WebPageTest: + +- A Private Instance has no API rate limiting, meaning you can use the API to test more than 200 page views per day (the limit on the public instance); +- WebPageTest audits results started through the API [will only be stored for 30 days](https://www.webpagetest.org/forums/showthread.php?tid=15759&pid=31710#pid31710) on the public instance, whereas you can store them indefinitely on your Private Instance; +- As your organization will likely be the sole user of the Private Instance, queuing times will be greatly reduced; +- A Private Instance can be hosted inside your own VPN, so that it will be able to access websites that are not accessible outside your network; +- It helps alleviate the load on the public instance 🙂 + +A few points to keep in mind if you’re planning to host your own Private Instance: + +- If you’re going to host it on AWS or GCP, it will cost you money. How much money will depend on your usage, as (at least on AWS) the WebPageTest test agents are spawned and killed automatically as you need them; +- Private Instances do not allow testing on mobile devices. + +To deploy your own WebPageTest Instance, follow the [documentation](https://github.com/WPO-Foundation/webpagetest-docs/blob/master/user/Private%20Instances/README.md). If you have any trouble, you might want to ask for help on the [WebPageTest Forums](https://www.webpagetest.org/forums/) or on the [WebPerformance Slack community](https://webperformance.herokuapp.com/), in the `#webpagetest-instances` channel. + +Once your Private Instance is up and running, you can setup Falco to use it! \ No newline at end of file diff --git a/docs/docs/wpt-private-instances/setting-up-falco-for-private-instances.md b/docs/docs/wpt-private-instances/setting-up-falco-for-private-instances.md new file mode 100644 index 000000000..60b291ae3 --- /dev/null +++ b/docs/docs/wpt-private-instances/setting-up-falco-for-private-instances.md @@ -0,0 +1,20 @@ +--- +id: setting-up-falco-for-private-instances +title: Setting up Falco for Private Instances +sidebar_label: Setting up Falco for Private Instances +--- + +Falco can be configure to use your Private Instance of WebPageTest to run audits, instead of the public one. This can be set up on a Project basis, so that some of your projects can use your Private Instance and other can use the public one—you can even use several Private Instances should you need to. + +To configure a Private Instance on a project, [make sure you are a Project Admin](/docs/getting-started/managing-users), and click on the “Manage project settings” link in the left-hand menu on that Project’s page. + +There, you should be able to enter your Private Instance URL (without the trailing slash) and the associated API Key that you declared during the Private Instance setup. If you get an error at that point, please make sure that: + +- The URL is correct; +- The API of the instance is correctly working (you can try to head to `/getLocations.php/getLocations.php?f=html`). + +In case you still encounter a problem, please [open an issue](https://github.com/theodo/falco/issues/new?template=bug.md) on the Falco repository! + +If everything went OK, you should see a green confirmation message. You can now head to the “Project environments” setting page and pick the environments you want to run the audits in. + +Falco will now run your audits against your Private instance 🎉 \ No newline at end of file diff --git a/docs/sidebars.js b/docs/sidebars.js index 7e1f3860b..c9fbabc1e 100755 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -21,5 +21,9 @@ module.exports = { 'api-reference/api-docs', 'api-reference/swagger-ui' ], + 'WebPageTest Private Instances': [ + 'wpt-private-instances/about-wpt-private-instances', + 'wpt-private-instances/setting-up-falco-for-private-instances' + ] }, };