From d147c969b542e599b5f58550fddffacca4f611d6 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 31 Jan 2024 13:05:54 -0600 Subject: [PATCH 01/76] Stop clearing search params for adv filters and search when switching between the two --- moped-editor/src/components/GridTable/Search.js | 2 -- moped-editor/src/components/GridTable/SearchBar.js | 6 ------ 2 files changed, 8 deletions(-) diff --git a/moped-editor/src/components/GridTable/Search.js b/moped-editor/src/components/GridTable/Search.js index 0e78f699e5..6b4e4eca66 100644 --- a/moped-editor/src/components/GridTable/Search.js +++ b/moped-editor/src/components/GridTable/Search.js @@ -95,8 +95,6 @@ const Search = ({ * Clears the filters when switching to simple search */ const handleSwitchToSearch = () => { - setFilters([]); - setIsOr(false); setAdvancedSearchAnchor(null); }; diff --git a/moped-editor/src/components/GridTable/SearchBar.js b/moped-editor/src/components/GridTable/SearchBar.js index b91b25bb87..6a5fa87671 100644 --- a/moped-editor/src/components/GridTable/SearchBar.js +++ b/moped-editor/src/components/GridTable/SearchBar.js @@ -17,10 +17,6 @@ import { import makeStyles from "@mui/styles/makeStyles"; import { Search as SearchIcon } from "react-feather"; import clsx from "clsx"; -import { - advancedSearchFilterParamName, - advancedSearchIsOrParamName, -} from "src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch"; import { simpleSearchParamName } from "src/views/projects/projectsListView/useProjectListViewQuery/useSearch"; /** @@ -153,8 +149,6 @@ const SearchBar = ({ // Update state to trigger search and set simple search param setSearchTerm(searchFieldValue); setSearchParams((prevSearchParams) => { - prevSearchParams.delete(advancedSearchFilterParamName); - prevSearchParams.delete(advancedSearchIsOrParamName); prevSearchParams.set(simpleSearchParamName, searchFieldValue); return prevSearchParams; }); From 609810ab0d5644522c8d745101a33203383df293 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 31 Jan 2024 13:27:35 -0600 Subject: [PATCH 02/76] Wrangle simple search reset into a function; stack simple and adv filters --- .../src/components/GridTable/Filters.js | 19 +++++++------- .../src/components/GridTable/Search.js | 25 +++++++++++-------- .../src/components/GridTable/SearchBar.js | 21 ++++------------ 3 files changed, 29 insertions(+), 36 deletions(-) diff --git a/moped-editor/src/components/GridTable/Filters.js b/moped-editor/src/components/GridTable/Filters.js index 6148318f74..fd85473c98 100644 --- a/moped-editor/src/components/GridTable/Filters.js +++ b/moped-editor/src/components/GridTable/Filters.js @@ -103,8 +103,7 @@ const useStyles = makeStyles((theme) => ({ * @param {Function} setFilters - Set the current filters from useAdvancedSearch hook * @param {Function} handleAdvancedSearchClose - Used to close the advanced search * @param {Object} filtersConfig - The configuration object for the filters - * @param {Function} setSearchFieldValue - Used to set the search field value - * @param {Function} setSearchTerm - Used to set the search term for simple search + * @param {Function} resetSimpleSearch - Function to reset the simple search * @return {JSX.Element} * @constructor */ @@ -113,8 +112,7 @@ const Filters = ({ setFilters, handleAdvancedSearchClose, filtersConfig, - setSearchFieldValue, - setSearchTerm, + resetSimpleSearch, isOr, setIsOr, }) => { @@ -264,13 +262,15 @@ const Filters = ({ }; /** - * Clears the filters + * Reset search box and advanced filters */ - const handleClearFilters = useCallback(() => { + const handleResetFilters = useCallback(() => { setFilterParameters([generateEmptyFilter()]); setFilters([]); setIsOr(false); + resetSimpleSearch(); + setSearchParams((prevSearchParams) => { prevSearchParams.delete(advancedSearchFilterParamName); prevSearchParams.delete(advancedSearchIsOrParamName); @@ -286,7 +286,6 @@ const Filters = ({ prevSearchParams.set(advancedSearchFilterParamName, jsonParamString); prevSearchParams.set(advancedSearchIsOrParamName, isOrToggleValue); - prevSearchParams.delete(simpleSearchParamName); return prevSearchParams; }); @@ -294,8 +293,8 @@ const Filters = ({ setFilters(filterParameters); handleAdvancedSearchClose(); // Clear simple search field in UI and state since we are using advanced search - setSearchFieldValue(""); - setSearchTerm(""); + // setSearchFieldValue(""); + // setSearchTerm(""); }; const handleAndOrToggleChange = (e) => { @@ -583,7 +582,7 @@ const Filters = ({ fullWidth variant="outlined" startIcon={} - onClick={handleClearFilters} + onClick={handleResetFilters} > Reset diff --git a/moped-editor/src/components/GridTable/Search.js b/moped-editor/src/components/GridTable/Search.js index 6b4e4eca66..c12b1dee74 100644 --- a/moped-editor/src/components/GridTable/Search.js +++ b/moped-editor/src/components/GridTable/Search.js @@ -1,11 +1,13 @@ import React, { useState } from "react"; import PropTypes from "prop-types"; +import { useSearchParams } from "react-router-dom"; import { Box, Button, Grid, Paper, Popper } from "@mui/material"; import SaveAltIcon from "@mui/icons-material/SaveAlt"; import Filters from "src/components/GridTable/Filters"; import SearchBar from "./SearchBar"; import makeStyles from "@mui/styles/makeStyles"; +import { simpleSearchParamName } from "src/views/projects/projectsListView/useProjectListViewQuery/useSearch"; const useStyles = makeStyles((theme) => ({ root: { @@ -84,6 +86,8 @@ const Search = ({ const classes = useStyles(); const divRef = React.useRef(); + let [, setSearchParams] = useSearchParams(); + /** * The contents of the search box in SearchBar * @type {string} searchFieldValue @@ -98,19 +102,11 @@ const Search = ({ setAdvancedSearchAnchor(null); }; - /** - * Clears the simple search when switching to advanced filters - */ - const handleSwitchToAdvancedSearch = () => { - setSearchTerm(""); - }; - const toggleAdvancedSearch = () => { if (advancedSearchAnchor) { setAdvancedSearchAnchor(null); } else { setAdvancedSearchAnchor(divRef.current); - handleSwitchToAdvancedSearch(); } }; @@ -118,6 +114,15 @@ const Search = ({ setAdvancedSearchAnchor(null); }; + const resetSimpleSearch = () => { + setSearchFieldValue(null); + setSearchTerm(null); + setSearchParams((prevSearchParams) => { + prevSearchParams.delete(simpleSearchParamName); + return prevSearchParams; + }); + }; + return (
@@ -142,6 +147,7 @@ const Search = ({ handleSwitchToSearch={handleSwitchToSearch} loading={loading} filtersConfig={filtersConfig} + resetSimpleSearch={resetSimpleSearch} /> diff --git a/moped-editor/src/components/GridTable/SearchBar.js b/moped-editor/src/components/GridTable/SearchBar.js index 6a5fa87671..bf94e47999 100644 --- a/moped-editor/src/components/GridTable/SearchBar.js +++ b/moped-editor/src/components/GridTable/SearchBar.js @@ -106,6 +106,7 @@ const SearchBar = ({ handleSwitchToSearch, loading, filtersConfig, + resetSimpleSearch, }) => { const classes = useStyles(); let [, setSearchParams] = useSearchParams(); @@ -124,7 +125,7 @@ const SearchBar = ({ const handleSearchValueChange = (value) => { if (value === "" && searchFieldValue !== "") { - handleClearSearchResults(); + resetSimpleSearch(); } else { setSearchFieldValue(value); } @@ -134,7 +135,7 @@ const SearchBar = ({ * Handles the submission of our search form * @param {Object} e - The event object */ - const handleSearchSubmission = (event) => { + const handleSearchSubmission = (event = null) => { // Stop if we don't have any value entered in the search field if (searchFieldValue.length === 0) { return; @@ -154,18 +155,6 @@ const SearchBar = ({ }); }; - /** - * Clears the search results - */ - const handleClearSearchResults = () => { - setSearchTerm(""); - setSearchFieldValue(""); - setSearchParams((prevSearchParams) => { - prevSearchParams.delete(simpleSearchParamName); - return prevSearchParams; - }); - }; - /** * Handles special keys typed in the search bar * @param {string} key - The key name being typed @@ -176,13 +165,13 @@ const SearchBar = ({ * On Escape key, clear the search */ case "Escape": - handleClearSearchResults(); + resetSimpleSearch(); break; /** * On Enter key, initialize the search */ case "Enter": - handleSearchSubmission(null); + handleSearchSubmission(); break; default: From 159969d2b53fa2e16576ceacfa7c59bec95f990f Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 31 Jan 2024 13:30:24 -0600 Subject: [PATCH 03/76] Fix console warnings --- moped-editor/src/components/GridTable/Filters.js | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/moped-editor/src/components/GridTable/Filters.js b/moped-editor/src/components/GridTable/Filters.js index fd85473c98..7d7669bc48 100644 --- a/moped-editor/src/components/GridTable/Filters.js +++ b/moped-editor/src/components/GridTable/Filters.js @@ -29,7 +29,6 @@ import { advancedSearchFilterParamName, advancedSearchIsOrParamName, } from "src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch"; -import { simpleSearchParamName } from "src/views/projects/projectsListView/useProjectListViewQuery/useSearch"; import { areAllFiltersComplete, checkIsValidInput, @@ -268,14 +267,13 @@ const Filters = ({ setFilterParameters([generateEmptyFilter()]); setFilters([]); setIsOr(false); - - resetSimpleSearch(); - setSearchParams((prevSearchParams) => { prevSearchParams.delete(advancedSearchFilterParamName); prevSearchParams.delete(advancedSearchIsOrParamName); }); - }, [setSearchParams, setFilters, setIsOr]); + + resetSimpleSearch(); + }, [setSearchParams, setFilters, setIsOr, resetSimpleSearch]); /** * Applies the current local state and updates the parent's state @@ -292,9 +290,6 @@ const Filters = ({ setIsOr(isOrToggleValue); setFilters(filterParameters); handleAdvancedSearchClose(); - // Clear simple search field in UI and state since we are using advanced search - // setSearchFieldValue(""); - // setSearchTerm(""); }; const handleAndOrToggleChange = (e) => { From 31e14ce8e28b582f733d4ae9725521936e19ef2e Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 31 Jan 2024 17:10:21 -0600 Subject: [PATCH 04/76] Remove handler for switch from adv filter to search --- .../src/components/GridTable/Search.js | 20 ++----------------- .../src/components/GridTable/SearchBar.js | 6 +----- .../useProjectListViewQuery/useSearch.js | 2 +- 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/moped-editor/src/components/GridTable/Search.js b/moped-editor/src/components/GridTable/Search.js index c12b1dee74..9c03f80157 100644 --- a/moped-editor/src/components/GridTable/Search.js +++ b/moped-editor/src/components/GridTable/Search.js @@ -1,13 +1,11 @@ import React, { useState } from "react"; import PropTypes from "prop-types"; -import { useSearchParams } from "react-router-dom"; import { Box, Button, Grid, Paper, Popper } from "@mui/material"; import SaveAltIcon from "@mui/icons-material/SaveAlt"; import Filters from "src/components/GridTable/Filters"; import SearchBar from "./SearchBar"; import makeStyles from "@mui/styles/makeStyles"; -import { simpleSearchParamName } from "src/views/projects/projectsListView/useProjectListViewQuery/useSearch"; const useStyles = makeStyles((theme) => ({ root: { @@ -86,8 +84,6 @@ const Search = ({ const classes = useStyles(); const divRef = React.useRef(); - let [, setSearchParams] = useSearchParams(); - /** * The contents of the search box in SearchBar * @type {string} searchFieldValue @@ -95,13 +91,6 @@ const Search = ({ */ const [searchFieldValue, setSearchFieldValue] = useState(searchTerm); - /** - * Clears the filters when switching to simple search - */ - const handleSwitchToSearch = () => { - setAdvancedSearchAnchor(null); - }; - const toggleAdvancedSearch = () => { if (advancedSearchAnchor) { setAdvancedSearchAnchor(null); @@ -115,12 +104,8 @@ const Search = ({ }; const resetSimpleSearch = () => { - setSearchFieldValue(null); - setSearchTerm(null); - setSearchParams((prevSearchParams) => { - prevSearchParams.delete(simpleSearchParamName); - return prevSearchParams; - }); + setSearchFieldValue(""); + setSearchTerm(""); }; return ( @@ -144,7 +129,6 @@ const Search = ({ setSearchTerm={setSearchTerm} queryConfig={queryConfig} isOr={isOr} - handleSwitchToSearch={handleSwitchToSearch} loading={loading} filtersConfig={filtersConfig} resetSimpleSearch={resetSimpleSearch} diff --git a/moped-editor/src/components/GridTable/SearchBar.js b/moped-editor/src/components/GridTable/SearchBar.js index bf94e47999..9fa1e718a4 100644 --- a/moped-editor/src/components/GridTable/SearchBar.js +++ b/moped-editor/src/components/GridTable/SearchBar.js @@ -103,7 +103,6 @@ const SearchBar = ({ setSearchTerm, queryConfig, isOr, - handleSwitchToSearch, loading, filtersConfig, resetSimpleSearch, @@ -135,7 +134,7 @@ const SearchBar = ({ * Handles the submission of our search form * @param {Object} e - The event object */ - const handleSearchSubmission = (event = null) => { + const handleSearchSubmission = (event) => { // Stop if we don't have any value entered in the search field if (searchFieldValue.length === 0) { return; @@ -144,9 +143,6 @@ const SearchBar = ({ // Prevent default behavior on any event if (event) event.preventDefault(); - // Clear the advanced search filters - handleSwitchToSearch(); - // Update state to trigger search and set simple search param setSearchTerm(searchFieldValue); setSearchParams((prevSearchParams) => { diff --git a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useSearch.js b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useSearch.js index 1aecb77b13..620f2efd97 100644 --- a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useSearch.js +++ b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useSearch.js @@ -38,7 +38,7 @@ export const useSearch = ({ queryConfig }) => { const [searchTerm, setSearchTerm] = useState(simpleSearchValue ?? ""); const searchWhereString = useMemo(() => { - if (searchTerm && searchTerm !== "") { + if (searchTerm && searchTerm.length > 0) { /** * Iterate through all column keys, if they are searchable * add the to the Or list. From e3d0fe95cdadecc591be65860586d55a483834e0 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 31 Jan 2024 17:22:11 -0600 Subject: [PATCH 05/76] Return prevSearchParams when setting so we don't lose them --- moped-editor/src/components/GridTable/Filters.js | 3 +++ moped-editor/src/components/GridTable/Search.js | 10 ++++++++++ moped-editor/src/components/GridTable/SearchBar.js | 1 + 3 files changed, 14 insertions(+) diff --git a/moped-editor/src/components/GridTable/Filters.js b/moped-editor/src/components/GridTable/Filters.js index 7d7669bc48..0b542a5b13 100644 --- a/moped-editor/src/components/GridTable/Filters.js +++ b/moped-editor/src/components/GridTable/Filters.js @@ -236,6 +236,7 @@ const Filters = ({ const jsonParamString = JSON.stringify(filtersNewState); setSearchParams((prevSearchParams) => { prevSearchParams.set(advancedSearchFilterParamName, jsonParamString); + return prevSearchParams; }); setFilterParameters(filtersNewState); @@ -270,6 +271,8 @@ const Filters = ({ setSearchParams((prevSearchParams) => { prevSearchParams.delete(advancedSearchFilterParamName); prevSearchParams.delete(advancedSearchIsOrParamName); + + return prevSearchParams; }); resetSimpleSearch(); diff --git a/moped-editor/src/components/GridTable/Search.js b/moped-editor/src/components/GridTable/Search.js index 9c03f80157..e7edab18f6 100644 --- a/moped-editor/src/components/GridTable/Search.js +++ b/moped-editor/src/components/GridTable/Search.js @@ -1,11 +1,13 @@ import React, { useState } from "react"; import PropTypes from "prop-types"; +import { useSearchParams } from "react-router-dom"; import { Box, Button, Grid, Paper, Popper } from "@mui/material"; import SaveAltIcon from "@mui/icons-material/SaveAlt"; import Filters from "src/components/GridTable/Filters"; import SearchBar from "./SearchBar"; import makeStyles from "@mui/styles/makeStyles"; +import { simpleSearchParamName } from "src/views/projects/projectsListView/useProjectListViewQuery/useSearch"; const useStyles = makeStyles((theme) => ({ root: { @@ -84,6 +86,8 @@ const Search = ({ const classes = useStyles(); const divRef = React.useRef(); + let [, setSearchParams] = useSearchParams(); + /** * The contents of the search box in SearchBar * @type {string} searchFieldValue @@ -106,6 +110,12 @@ const Search = ({ const resetSimpleSearch = () => { setSearchFieldValue(""); setSearchTerm(""); + + setSearchParams((prevSearchParams) => { + prevSearchParams.delete(simpleSearchParamName); + + return prevSearchParams; + }); }; return ( diff --git a/moped-editor/src/components/GridTable/SearchBar.js b/moped-editor/src/components/GridTable/SearchBar.js index 9fa1e718a4..ffd1e023ca 100644 --- a/moped-editor/src/components/GridTable/SearchBar.js +++ b/moped-editor/src/components/GridTable/SearchBar.js @@ -147,6 +147,7 @@ const SearchBar = ({ setSearchTerm(searchFieldValue); setSearchParams((prevSearchParams) => { prevSearchParams.set(simpleSearchParamName, searchFieldValue); + return prevSearchParams; }); }; From 9ef28556b5867e794b8b56cfd5ccfab9ccacaf41 Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Thu, 1 Feb 2024 14:04:30 -0600 Subject: [PATCH 06/76] begin updating fields --- moped-database/metadata/tables.yaml | 30 ++++++++++++------- .../down.sql | 1 + .../up.sql | 1 + 3 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql create mode 100644 moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index 073057a62b..650a26fcd4 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -2638,9 +2638,12 @@ - role: moped-admin permission: check: {} + set: + created_by_user_id: x-hasura-user-db-id + updated_by_user_id: x-hasura-user-db-id columns: - completed - - date_added + - created_at - milestone_id - description - date_actual @@ -2648,12 +2651,16 @@ - milestone_order - project_id - is_deleted + comment: No insert permissions on audit fields - role: moped-editor permission: check: {} + set: + created_by_user_id: x-hasura-user-db-id + updated_by_user_id: x-hasura-user-db-id columns: - completed - - date_added + - created_at - milestone_id - description - date_actual @@ -2661,12 +2668,13 @@ - milestone_order - project_id - is_deleted + comment: No insert permissions on audit fields select_permissions: - role: moped-admin permission: columns: - completed - - date_added + - created_at - milestone_id - description - date_actual @@ -2680,7 +2688,7 @@ permission: columns: - completed - - date_added + - created_at - milestone_id - description - date_actual @@ -2694,7 +2702,7 @@ permission: columns: - completed - - date_added + - created_at - milestone_id - description - date_actual @@ -2709,7 +2717,7 @@ permission: columns: - completed - - date_added + - created_at - milestone_id - description - date_actual @@ -2723,7 +2731,7 @@ permission: columns: - completed - - date_added + - created_at - milestone_id - description - date_actual @@ -3413,8 +3421,6 @@ update_permissions: - role: moped-admin permission: - set: - updated_by_user_id: x-hasura-user-db-id columns: - is_current_phase - is_deleted @@ -3428,10 +3434,10 @@ - is_phase_end_confirmed filter: {} check: {} - - role: moped-editor - permission: set: updated_by_user_id: x-hasura-user-db-id + - role: moped-editor + permission: columns: - is_current_phase - is_deleted @@ -3445,6 +3451,8 @@ - is_phase_end_confirmed filter: {} check: {} + set: + updated_by_user_id: x-hasura-user-db-id event_triggers: - name: activity_log_moped_proj_phases definition: diff --git a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql new file mode 100644 index 0000000000..16758402c6 --- /dev/null +++ b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql @@ -0,0 +1 @@ +alter table "public"."moped_proj_milestones" rename column "created_at" to "date_added"; diff --git a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql new file mode 100644 index 0000000000..f1334fbebf --- /dev/null +++ b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql @@ -0,0 +1 @@ +alter table "public"."moped_proj_milestones" rename column "date_added" to "created_at"; From 994ffc3c28dbb555dcc90315504ab520f9d73027 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 2 Feb 2024 12:22:10 -0600 Subject: [PATCH 07/76] Start script to find orphan Knack records --- .../find_orphan_knack_project_records.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 moped-etl/data-tracker-sync/find_orphan_knack_project_records.py diff --git a/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py b/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py new file mode 100644 index 0000000000..2efb1587b7 --- /dev/null +++ b/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +import os +import argparse +import logging +from datetime import datetime, timezone + +import knackpy + +from process.request import make_hasura_request +from process.logging import get_logger + +KNACK_DATA_TRACKER_APP_ID = os.getenv("KNACK_DATA_TRACKER_APP_ID") +KNACK_DATA_TRACKER_API_KEY = os.getenv("KNACK_DATA_TRACKER_API_KEY") + +KNACK_DATA_TRACKER_PROJECT_OBJECT = "object_201" + +GET_MOPED_PROJECTS = """ +query GetMopedProjects { + moped_project(where: { knack_project_id: { _is_null: false }}) { + project_id + knack_project_id + } +} +""" + + +# def create_knack_project_from_moped_project(moped_project_record, is_test=False): +# """ +# Create a Knack project record to sync a Moped project to Data Tracker + +# Parameters: +# moped_project_record (dict): A Moped project record +# is_test (boolean): test flag added to add a complatible Knack signal record id to payload + +# Returns: +# String: Knack record ID of created record +# """ +# knack_project_record = build_knack_project_from_moped_project( +# moped_project_record=moped_project_record, is_test=is_test +# ) + +# created = knackpy.api.record( +# app_id=KNACK_DATA_TRACKER_APP_ID, +# api_key=KNACK_DATA_TRACKER_API_KEY, +# method="create", +# data=knack_project_record, +# obj=KNACK_DATA_TRACKER_PROJECT_OBJECT, +# ) + +# logger.debug(f"Created Knack record: {created}") +# knack_record_id = created["id"] +# return knack_record_id + + +def get_synced_moped_project_knack_ids(): + print("get_synced_moped_project_knack_ids") + data = make_hasura_request(query=GET_MOPED_PROJECTS) + projects = data["moped_project"] + + knack_ids = [project["knack_project_id"] for project in projects] + print(f"Knack IDs: {knack_ids}") + print(f"Found {len(knack_ids)} Knack IDs") + + +def get_knack_record_ids_not_synced_to_moped(): + # TODO: gather knack record ids no in list collected by get_synced_moped_project_knack_ids + print("get_knack_record_ids_not_synced_to_moped") + + +def main(args): + get_synced_moped_project_knack_ids() + + # logger.info(f"Done syncing.") + # logger.info(f"Created {len(created_knack_records)} new Knack records") + # logger.debug(f"Records created: {created_knack_records}") + # logger.info(f"Updated {len(updated_knack_records)} existing Knack records") + # logger.debug(f"Updated Knack records: {updated_knack_records}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("-t", "--test", action="store_true") + + args = parser.parse_args() + + log_level = logging.DEBUG + logger = get_logger(name="knack-orphan-records", level=log_level) + logger.info(f"Starting.") + + main(args) From 862432b4c305808d91c49510af54f6bb26556df8 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 2 Feb 2024 12:33:41 -0600 Subject: [PATCH 08/76] Get knack project table ids --- .../find_orphan_knack_project_records.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py b/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py index 2efb1587b7..e76ddd6e83 100644 --- a/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py +++ b/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py @@ -54,22 +54,30 @@ def get_synced_moped_project_knack_ids(): - print("get_synced_moped_project_knack_ids") data = make_hasura_request(query=GET_MOPED_PROJECTS) projects = data["moped_project"] - knack_ids = [project["knack_project_id"] for project in projects] - print(f"Knack IDs: {knack_ids}") - print(f"Found {len(knack_ids)} Knack IDs") + logger.debug(f"Found Knack IDs: {knack_ids}") + logger.info(f"Found {len(knack_ids)} Knack IDs") + return knack_ids + + +def get_knack_project_record_ids(): + knack_projects = knackpy.api.get( + app_id=KNACK_DATA_TRACKER_APP_ID, + api_key=KNACK_DATA_TRACKER_API_KEY, + obj=KNACK_DATA_TRACKER_PROJECT_OBJECT, + ) + knack_ids = [project["id"] for project in knack_projects] -def get_knack_record_ids_not_synced_to_moped(): - # TODO: gather knack record ids no in list collected by get_synced_moped_project_knack_ids - print("get_knack_record_ids_not_synced_to_moped") + logger.debug(f"Found Knack projects: {knack_ids}") + logger.info(f"Found {len(knack_ids)} Knack projects") def main(args): - get_synced_moped_project_knack_ids() + knack_project_ids_in_moped = get_synced_moped_project_knack_ids() + knack_project_ids_knack = get_knack_project_record_ids() # logger.info(f"Done syncing.") # logger.info(f"Created {len(created_knack_records)} new Knack records") From cce9c2023f61086342376179b159ddce8d882d40 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 2 Feb 2024 12:50:49 -0600 Subject: [PATCH 09/76] Get intersection and difference in moped and knack project id lists --- .../find_orphan_knack_project_records.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py b/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py index e76ddd6e83..851d3f2d3e 100644 --- a/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py +++ b/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py @@ -70,14 +70,31 @@ def get_knack_project_record_ids(): obj=KNACK_DATA_TRACKER_PROJECT_OBJECT, ) knack_ids = [project["id"] for project in knack_projects] + logger.info(knack_projects[0]) logger.debug(f"Found Knack projects: {knack_ids}") logger.info(f"Found {len(knack_ids)} Knack projects") + return knack_ids def main(args): knack_project_ids_in_moped = get_synced_moped_project_knack_ids() - knack_project_ids_knack = get_knack_project_record_ids() + knack_project_ids_in_knack = get_knack_project_record_ids() + + ids_in_both_tables = list( + set(knack_project_ids_in_knack) & set(knack_project_ids_in_moped) + ) + ids_not_in_both_tables = list( + set(knack_project_ids_in_knack) - set(knack_project_ids_in_moped) + ) + + logger.info(f"Found {len(ids_in_both_tables)} records in both tables") + # logger.debug(f"IDs in both tables: {ids_in_both_tables}") + + logger.info( + f"Found {len(ids_not_in_both_tables)} records in Knack but not in Moped" + ) + # logger.debug(f"IDs in Knack but not in Moped: {ids_not_in_both_tables}") # logger.info(f"Done syncing.") # logger.info(f"Created {len(created_knack_records)} new Knack records") @@ -93,7 +110,7 @@ def main(args): args = parser.parse_args() - log_level = logging.DEBUG + log_level = logging.INFO logger = get_logger(name="knack-orphan-records", level=log_level) logger.info(f"Starting.") From c26c36807a927c0d5ecdcb85581af8ff30271b60 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 2 Feb 2024 17:59:19 -0600 Subject: [PATCH 10/76] Get work order signals connected to projects table records --- moped-etl/data-tracker-sync/.gitignore | 1 + .../find_orphan_knack_project_records.py | 84 +++++++++---------- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/moped-etl/data-tracker-sync/.gitignore b/moped-etl/data-tracker-sync/.gitignore index 993ea7bd1f..9aa2c776cb 100644 --- a/moped-etl/data-tracker-sync/.gitignore +++ b/moped-etl/data-tracker-sync/.gitignore @@ -1 +1,2 @@ *env +env_file_production diff --git a/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py b/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py index 851d3f2d3e..5a5599620d 100644 --- a/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py +++ b/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py @@ -14,6 +14,9 @@ KNACK_DATA_TRACKER_API_KEY = os.getenv("KNACK_DATA_TRACKER_API_KEY") KNACK_DATA_TRACKER_PROJECT_OBJECT = "object_201" +KNACK_DATA_TRACKER_WORK_ORDER_SIGNALS_OBJECT = "object_31" + +WORK_ORDER_SIGNALS_PROJECT_FIELD = "field_3965" GET_MOPED_PROJECTS = """ query GetMopedProjects { @@ -25,41 +28,13 @@ """ -# def create_knack_project_from_moped_project(moped_project_record, is_test=False): -# """ -# Create a Knack project record to sync a Moped project to Data Tracker - -# Parameters: -# moped_project_record (dict): A Moped project record -# is_test (boolean): test flag added to add a complatible Knack signal record id to payload - -# Returns: -# String: Knack record ID of created record -# """ -# knack_project_record = build_knack_project_from_moped_project( -# moped_project_record=moped_project_record, is_test=is_test -# ) - -# created = knackpy.api.record( -# app_id=KNACK_DATA_TRACKER_APP_ID, -# api_key=KNACK_DATA_TRACKER_API_KEY, -# method="create", -# data=knack_project_record, -# obj=KNACK_DATA_TRACKER_PROJECT_OBJECT, -# ) - -# logger.debug(f"Created Knack record: {created}") -# knack_record_id = created["id"] -# return knack_record_id - - def get_synced_moped_project_knack_ids(): data = make_hasura_request(query=GET_MOPED_PROJECTS) projects = data["moped_project"] knack_ids = [project["knack_project_id"] for project in projects] - logger.debug(f"Found Knack IDs: {knack_ids}") - logger.info(f"Found {len(knack_ids)} Knack IDs") + logger.debug(f"Found Moped Project Knack IDs: {knack_ids}") + logger.info(f"Found {len(knack_ids)} Moped Project Knack IDs") return knack_ids @@ -70,13 +45,31 @@ def get_knack_project_record_ids(): obj=KNACK_DATA_TRACKER_PROJECT_OBJECT, ) knack_ids = [project["id"] for project in knack_projects] - logger.info(knack_projects[0]) logger.debug(f"Found Knack projects: {knack_ids}") logger.info(f"Found {len(knack_ids)} Knack projects") return knack_ids +def check_for_work_order_signals_connection(knack_id): + work_orders = knackpy.api.get( + app_id=KNACK_DATA_TRACKER_APP_ID, + api_key=KNACK_DATA_TRACKER_API_KEY, + obj=KNACK_DATA_TRACKER_WORK_ORDER_SIGNALS_OBJECT, + filters=[ + { + "field": WORK_ORDER_SIGNALS_PROJECT_FIELD, + "operator": "is", + "value": knack_id, + } + ], + ) + + logger.debug(f"Found Knack work orders: {work_orders}") + logger.info(f"Found {len(work_orders)} Knack work orders") + return work_orders + + def main(args): knack_project_ids_in_moped = get_synced_moped_project_knack_ids() knack_project_ids_in_knack = get_knack_project_record_ids() @@ -84,23 +77,30 @@ def main(args): ids_in_both_tables = list( set(knack_project_ids_in_knack) & set(knack_project_ids_in_moped) ) + logger.info(f"Found {len(ids_in_both_tables)} records in both tables") + ids_not_in_both_tables = list( set(knack_project_ids_in_knack) - set(knack_project_ids_in_moped) ) - - logger.info(f"Found {len(ids_in_both_tables)} records in both tables") - # logger.debug(f"IDs in both tables: {ids_in_both_tables}") - logger.info( f"Found {len(ids_not_in_both_tables)} records in Knack but not in Moped" ) - # logger.debug(f"IDs in Knack but not in Moped: {ids_not_in_both_tables}") - # logger.info(f"Done syncing.") - # logger.info(f"Created {len(created_knack_records)} new Knack records") - # logger.debug(f"Records created: {created_knack_records}") - # logger.info(f"Updated {len(updated_knack_records)} existing Knack records") - # logger.debug(f"Updated Knack records: {updated_knack_records}") + deletes_to_skip = [] + for id in ids_not_in_both_tables: + project = check_for_work_order_signals_connection(id) + logger.debug( + f"Found work order signals connected to project ID {id}: {project}" + ) + if len(project) > 0: + deletes_to_skip.append(id) + + logger.info(f"Found {len(deletes_to_skip)} records to skip deleting") + logger.debug(f"Records to skip deleting: {deletes_to_skip}") + + # TODO: Do the delete part + + logger.info(f"Done.") if __name__ == "__main__": @@ -110,7 +110,7 @@ def main(args): args = parser.parse_args() - log_level = logging.INFO + log_level = logging.DEBUG logger = get_logger(name="knack-orphan-records", level=log_level) logger.info(f"Starting.") From 76cace60f02884d953fff58bccd4632f5bd68749 Mon Sep 17 00:00:00 2001 From: John Clary Date: Fri, 2 Feb 2024 22:54:06 -0500 Subject: [PATCH 11/76] (re)bump package and hasura versions --- moped-database/docker-compose.yml | 2 +- moped-editor/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-database/docker-compose.yml b/moped-database/docker-compose.yml index e43fea5c9b..db66e8785a 100644 --- a/moped-database/docker-compose.yml +++ b/moped-database/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.7' services: hasura: - image: hasura/graphql-engine:v2.36.2 + image: hasura/graphql-engine:v2.37.0 restart: always depends_on: - moped-pgsql diff --git a/moped-editor/package.json b/moped-editor/package.json index c7099c6b50..fc0926fa03 100644 --- a/moped-editor/package.json +++ b/moped-editor/package.json @@ -2,7 +2,7 @@ "name": "atd-moped-editor", "author": "ATD Data & Technology Services", "license": "CC0-1.0", - "version": "2.4.1", + "version": "2.5.0", "homepage": "/moped", "private": false, "repository": { From 7f489c4bf5fbd32741dee596aeea5af570b6edf3 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 5 Feb 2024 12:56:42 -0600 Subject: [PATCH 12/76] Add blurb to readme about sync eval script --- moped-etl/arcgis/README.md | 11 ++++++ ..._project_records.py => sync_evaluation.py} | 37 +++++++++++++------ 2 files changed, 37 insertions(+), 11 deletions(-) rename moped-etl/data-tracker-sync/{find_orphan_knack_project_records.py => sync_evaluation.py} (72%) diff --git a/moped-etl/arcgis/README.md b/moped-etl/arcgis/README.md index 9b495912c3..46225b98d4 100644 --- a/moped-etl/arcgis/README.md +++ b/moped-etl/arcgis/README.md @@ -30,3 +30,14 @@ or, to mount your local copy to a Docker container ```shell docker run -it --rm --network host --env-file env_file -v ${PWD}:/app atddocker/atd-moped-etl-arcgis:production python components_to_agol.py ``` + +### Sync evaluation script + +On first deployment of this ETL, duplicate records were created in the Knack `projects` table. The `sync_evaluation.py` script: + +- Gathers all Knack project record IDs stored in the `moped_project` table rows in the `knack_project_id` column +- Gather all Knack record IDs of all rows in the Data Tracker `projects` table +- Evaluates the overlap (synced correctly) and difference (mark for deletion) of these two lists +- Evaluates the list of differences to make sure there are no connections between Knack records marked for deletion and `work_order_signals` records +- Deletes the records with no connections and logs records with connections that were not deleted + \ No newline at end of file diff --git a/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py b/moped-etl/data-tracker-sync/sync_evaluation.py similarity index 72% rename from moped-etl/data-tracker-sync/find_orphan_knack_project_records.py rename to moped-etl/data-tracker-sync/sync_evaluation.py index 5a5599620d..7658f8803f 100644 --- a/moped-etl/data-tracker-sync/find_orphan_knack_project_records.py +++ b/moped-etl/data-tracker-sync/sync_evaluation.py @@ -52,7 +52,7 @@ def get_knack_project_record_ids(): def check_for_work_order_signals_connection(knack_id): - work_orders = knackpy.api.get( + work_order_signals = knackpy.api.get( app_id=KNACK_DATA_TRACKER_APP_ID, api_key=KNACK_DATA_TRACKER_API_KEY, obj=KNACK_DATA_TRACKER_WORK_ORDER_SIGNALS_OBJECT, @@ -65,38 +65,53 @@ def check_for_work_order_signals_connection(knack_id): ], ) - logger.debug(f"Found Knack work orders: {work_orders}") - logger.info(f"Found {len(work_orders)} Knack work orders") - return work_orders + logger.debug(f"Found Knack work order signals record: {work_order_signals}") + return work_order_signals + + +def delete_knack_project_record(knack_id): + logger.info(f"Deleted Knack project record with ID: {knack_id}") def main(args): + logger.info(f"Getting all Knack project IDs from Moped projects...") knack_project_ids_in_moped = get_synced_moped_project_knack_ids() + logger.info(f"Getting all Knack project IDs from Knack...") knack_project_ids_in_knack = get_knack_project_record_ids() + logger.info(f"Finding overlap and differences in those lists...") ids_in_both_tables = list( set(knack_project_ids_in_knack) & set(knack_project_ids_in_moped) ) - logger.info(f"Found {len(ids_in_both_tables)} records in both tables") + logger.info( + f"Found {len(ids_in_both_tables)} records in both tables that should be retained" + ) ids_not_in_both_tables = list( set(knack_project_ids_in_knack) - set(knack_project_ids_in_moped) ) + logger.info(f"Records in Knack but not in Moped: {ids_not_in_both_tables}") logger.info( - f"Found {len(ids_not_in_both_tables)} records in Knack but not in Moped" + f"Found {len(ids_not_in_both_tables)} records in Knack but not in Moped that are ready to delete" ) + logger.info(f"Checking Knack project records for work order signals connections...") deletes_to_skip = [] + count = 1 for id in ids_not_in_both_tables: - project = check_for_work_order_signals_connection(id) + logger.info(f"{count}/{len(ids_not_in_both_tables)}") + + work_order_signals = check_for_work_order_signals_connection(id) logger.debug( - f"Found work order signals connected to project ID {id}: {project}" + f"Found work order signals connected to project ID {id}: {work_order_signals}" ) - if len(project) > 0: + count += 1 + + if len(work_order_signals) > 0: deletes_to_skip.append(id) logger.info(f"Found {len(deletes_to_skip)} records to skip deleting") - logger.debug(f"Records to skip deleting: {deletes_to_skip}") + logger.info(f"Records to skip deleting: {deletes_to_skip}") # TODO: Do the delete part @@ -110,7 +125,7 @@ def main(args): args = parser.parse_args() - log_level = logging.DEBUG + log_level = logging.INFO logger = get_logger(name="knack-orphan-records", level=log_level) logger.info(f"Starting.") From daed3e696f38ec0bc109f820cbe24448581787f6 Mon Sep 17 00:00:00 2001 From: John Clary Date: Mon, 5 Feb 2024 14:28:32 -0500 Subject: [PATCH 13/76] use null instead of empty strings --- .../projects/projectView/ProjectComponents/ComponentForm.js | 6 +++--- .../projectView/ProjectComponents/CreateComponentModal.js | 4 ++-- .../projectView/ProjectComponents/EditAttributesModal.js | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/moped-editor/src/views/projects/projectView/ProjectComponents/ComponentForm.js b/moped-editor/src/views/projects/projectView/ProjectComponents/ComponentForm.js index 0e13e20842..3bbbc2e22a 100644 --- a/moped-editor/src/views/projects/projectView/ProjectComponents/ComponentForm.js +++ b/moped-editor/src/views/projects/projectView/ProjectComponents/ComponentForm.js @@ -42,11 +42,11 @@ const defaultFormValues = { subphase: null, tags: [], completionDate: null, - locationDescription: "", - description: "", + locationDescription: null, + description: null, work_types: [DEFAULT_COMPONENT_WORK_TYPE_OPTION], signal: null, - srtsId: "", + srtsId: null, }; const validationSchema = yup.object().shape({ diff --git a/moped-editor/src/views/projects/projectView/ProjectComponents/CreateComponentModal.js b/moped-editor/src/views/projects/projectView/ProjectComponents/CreateComponentModal.js index aa1b5c8770..ee5dfc4d1a 100644 --- a/moped-editor/src/views/projects/projectView/ProjectComponents/CreateComponentModal.js +++ b/moped-editor/src/views/projects/projectView/ProjectComponents/CreateComponentModal.js @@ -49,8 +49,8 @@ const CreateComponentModal = ({ moped_subcomponents: subcomponents, work_types, description, - phase_id: phase?.data.phase_id, - subphase_id: subphase?.data.subphase_id, + phase_id: phase?.data.phase_id || null, + subphase_id: subphase?.data.subphase_id || null, completion_date: completionDate, label: component_name, features: [], diff --git a/moped-editor/src/views/projects/projectView/ProjectComponents/EditAttributesModal.js b/moped-editor/src/views/projects/projectView/ProjectComponents/EditAttributesModal.js index 87c1c7056e..72dab6a089 100644 --- a/moped-editor/src/views/projects/projectView/ProjectComponents/EditAttributesModal.js +++ b/moped-editor/src/views/projects/projectView/ProjectComponents/EditAttributesModal.js @@ -134,14 +134,16 @@ const EditAttributesModal = ({ subphase: makeSubphaseFormFieldValue(clickedComponent.moped_subphase), completionDate: clickedComponent.completion_date, srtsId: - clickedComponent.srts_id?.length > 0 ? clickedComponent.srts_id : "", + clickedComponent.srts_id?.length > 0 + ? clickedComponent.srts_id + : null, tags: makeTagFormFieldValues( clickedComponent.moped_proj_component_tags ), locationDescription: clickedComponent.location_description?.length > 0 ? clickedComponent.location_description - : "", + : null, councilDistrict: !!clickedComponent.council_districts[0] ? clickedComponent.council_districts.join(", ") : "-", From 59bbbd90bde34c3527f70265a54ad480f59be4f2 Mon Sep 17 00:00:00 2001 From: John Clary Date: Mon, 5 Feb 2024 14:34:28 -0500 Subject: [PATCH 14/76] create migra to set empty component attributes to null --- .../down.sql | 2 ++ .../up.sql | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 moped-database/migrations/1707160404494_clear_component_empty_strings/down.sql create mode 100644 moped-database/migrations/1707160404494_clear_component_empty_strings/up.sql diff --git a/moped-database/migrations/1707160404494_clear_component_empty_strings/down.sql b/moped-database/migrations/1707160404494_clear_component_empty_strings/down.sql new file mode 100644 index 0000000000..c866df67d4 --- /dev/null +++ b/moped-database/migrations/1707160404494_clear_component_empty_strings/down.sql @@ -0,0 +1,2 @@ +-- can't undo ;) +SELECT 0; diff --git a/moped-database/migrations/1707160404494_clear_component_empty_strings/up.sql b/moped-database/migrations/1707160404494_clear_component_empty_strings/up.sql new file mode 100644 index 0000000000..bcd6c39c96 --- /dev/null +++ b/moped-database/migrations/1707160404494_clear_component_empty_strings/up.sql @@ -0,0 +1,26 @@ +-- temporarily disable event triggers + +SET session_replication_role = replica; +UPDATE + moped_proj_components +SET + description = NULL +WHERE + description = ''; + +UPDATE + moped_proj_components +SET + location_description = NULL +WHERE + location_description = ''; + +UPDATE + moped_proj_components +SET + srts_id = NULL +WHERE + srts_id = ''; + +-- renable event triggers +SET session_replication_role = DEFAULT; From 06fce1b06bf10d6d0a49137f5f03aeec435a50c2 Mon Sep 17 00:00:00 2001 From: John Clary Date: Mon, 5 Feb 2024 14:38:53 -0500 Subject: [PATCH 15/76] fix linebreak --- .../1707160404494_clear_component_empty_strings/up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-database/migrations/1707160404494_clear_component_empty_strings/up.sql b/moped-database/migrations/1707160404494_clear_component_empty_strings/up.sql index bcd6c39c96..51c2cdb11e 100644 --- a/moped-database/migrations/1707160404494_clear_component_empty_strings/up.sql +++ b/moped-database/migrations/1707160404494_clear_component_empty_strings/up.sql @@ -1,6 +1,6 @@ -- temporarily disable event triggers - SET session_replication_role = replica; + UPDATE moped_proj_components SET From 7ab06ffb0ebddeb747b9d50ec53992a53d4836f2 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 5 Feb 2024 14:24:01 -0600 Subject: [PATCH 16/76] Do the deletes and remove argparse --- .../data-tracker-sync/sync_evaluation.py | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/moped-etl/data-tracker-sync/sync_evaluation.py b/moped-etl/data-tracker-sync/sync_evaluation.py index 7658f8803f..aabfc1632f 100644 --- a/moped-etl/data-tracker-sync/sync_evaluation.py +++ b/moped-etl/data-tracker-sync/sync_evaluation.py @@ -70,10 +70,17 @@ def check_for_work_order_signals_connection(knack_id): def delete_knack_project_record(knack_id): - logger.info(f"Deleted Knack project record with ID: {knack_id}") + logger.info(f"Deleting Knack project record with ID: {knack_id}") + knackpy.api.record( + app_id=KNACK_DATA_TRACKER_APP_ID, + api_key=KNACK_DATA_TRACKER_API_KEY, + obj=KNACK_DATA_TRACKER_PROJECT_OBJECT, + method="delete", + data={"id": knack_id}, + ) -def main(args): +def main(): logger.info(f"Getting all Knack project IDs from Moped projects...") knack_project_ids_in_moped = get_synced_moped_project_knack_ids() logger.info(f"Getting all Knack project IDs from Knack...") @@ -99,34 +106,29 @@ def main(args): deletes_to_skip = [] count = 1 for id in ids_not_in_both_tables: - logger.info(f"{count}/{len(ids_not_in_both_tables)}") + logger.info(f"{count}/{len(ids_not_in_both_tables)}: Knack ID {id}") work_order_signals = check_for_work_order_signals_connection(id) - logger.debug( - f"Found work order signals connected to project ID {id}: {work_order_signals}" - ) count += 1 if len(work_order_signals) > 0: + logger.info( + f"Found work order signals connected to project ID {id}: {work_order_signals}" + ) deletes_to_skip.append(id) + else: + logger.info(f"No work order signals connection found deleting...") + delete_knack_project_record(id) - logger.info(f"Found {len(deletes_to_skip)} records to skip deleting") - logger.info(f"Records to skip deleting: {deletes_to_skip}") - - # TODO: Do the delete part - + logger.info( + f"Record IDs of {len(deletes_to_skip)} deletes skipped: {deletes_to_skip}" + ) logger.info(f"Done.") if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument("-t", "--test", action="store_true") - - args = parser.parse_args() - log_level = logging.INFO logger = get_logger(name="knack-orphan-records", level=log_level) logger.info(f"Starting.") - main(args) + main() From cb6f131288f829d2ee8d877c19c747a0fbd54e74 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 5 Feb 2024 15:57:27 -0600 Subject: [PATCH 17/76] Add my env file to dockerignore; remove unused dependencies --- moped-etl/data-tracker-sync/.dockerignore | 1 + moped-etl/data-tracker-sync/sync_evaluation.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/moped-etl/data-tracker-sync/.dockerignore b/moped-etl/data-tracker-sync/.dockerignore index ebfcfbe643..e400cc8ca6 100644 --- a/moped-etl/data-tracker-sync/.dockerignore +++ b/moped-etl/data-tracker-sync/.dockerignore @@ -1,3 +1,4 @@ env_file +env_file_production .git% __pycache__ diff --git a/moped-etl/data-tracker-sync/sync_evaluation.py b/moped-etl/data-tracker-sync/sync_evaluation.py index aabfc1632f..2de6d92c42 100644 --- a/moped-etl/data-tracker-sync/sync_evaluation.py +++ b/moped-etl/data-tracker-sync/sync_evaluation.py @@ -1,9 +1,7 @@ #!/usr/bin/env python import os -import argparse import logging -from datetime import datetime, timezone import knackpy @@ -128,7 +126,7 @@ def main(): if __name__ == "__main__": log_level = logging.INFO - logger = get_logger(name="knack-orphan-records", level=log_level) + logger = get_logger(name="sync_evaluation", level=log_level) logger.info(f"Starting.") main() From c6bec4ee1ad81bd87101aa4f5dcdbafd5ce18e6e Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 6 Feb 2024 13:52:12 -0600 Subject: [PATCH 18/76] Fix sync ETL output comment --- moped-etl/data-tracker-sync/data_tracker_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-etl/data-tracker-sync/data_tracker_sync.py b/moped-etl/data-tracker-sync/data_tracker_sync.py index 50e94b56c5..6aaffde8d7 100644 --- a/moped-etl/data-tracker-sync/data_tracker_sync.py +++ b/moped-etl/data-tracker-sync/data_tracker_sync.py @@ -194,7 +194,7 @@ def main(args): # Update Moped project with Knack record ID of created record logger.info( - f"Updating Knack ID {knack_record_id} from Moped project {moped_project_id}" + f"Updating Moped project {moped_project_id} with Knack ID {knack_record_id}" ) update_moped_project_knack_id(moped_project_id, knack_record_id) From d5860d68b70be6d58e50ab70b677c116f5f53ffb Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 6 Feb 2024 16:59:16 -0600 Subject: [PATCH 19/76] Remove last filter and enable apply only the simple search --- .../src/components/GridTable/Filters.js | 105 ++++++++++-------- .../src/components/GridTable/helpers.js | 2 +- 2 files changed, 59 insertions(+), 48 deletions(-) diff --git a/moped-editor/src/components/GridTable/Filters.js b/moped-editor/src/components/GridTable/Filters.js index 0b542a5b13..95242d49d2 100644 --- a/moped-editor/src/components/GridTable/Filters.js +++ b/moped-editor/src/components/GridTable/Filters.js @@ -15,6 +15,7 @@ import { Icon, IconButton, Grow, + Typography, } from "@mui/material"; import RadioGroup from "@mui/material/RadioGroup"; import Radio from "@mui/material/Radio"; @@ -227,23 +228,33 @@ const Filters = ({ * @param {string} filterIndex - The index of the filter to be deleted */ const handleDeleteFilterButtonClick = (filterIndex) => { - // Clone the state + // Clone the state, delete the filter index of the button clicked const filtersNewState = [...filterParameters]; - - // Delete the key (if it's there) filtersNewState.splice(filterIndex, 1); - // Finally, reset the state - const jsonParamString = JSON.stringify(filtersNewState); - setSearchParams((prevSearchParams) => { - prevSearchParams.set(advancedSearchFilterParamName, jsonParamString); - return prevSearchParams; - }); - setFilterParameters(filtersNewState); + const remainingFiltersCount = filtersNewState.length; + + if (remainingFiltersCount === 0) { + setFilterParameters([]); + setSearchParams((prevSearchParams) => { + prevSearchParams.delete(advancedSearchFilterParamName); + prevSearchParams.delete(advancedSearchIsOrParamName); - /* Reset isOr to false (all/and) if there is only one filter left */ - if (Object.keys(filtersNewState).length === 1) { + return prevSearchParams; + }); + } else if (remainingFiltersCount === 1) { + /* Reset isOr to false (all/and) if there is only one filter left */ + setFilterParameters(filtersNewState); setIsOrToggleValue(false); + } else { + // Finally, reset the state + setFilterParameters(filtersNewState); + const jsonParamString = JSON.stringify(filtersNewState); + setSearchParams((prevSearchParams) => { + prevSearchParams.set(advancedSearchFilterParamName, jsonParamString); + + return prevSearchParams; + }); } }; @@ -353,7 +364,15 @@ const Filters = ({ - + {filterParameters.length === 0 ? ( + + + + No filters applied + + + + ) : null} {filterParameters.map((filter, filterIndex) => { const { field: fieldName, operator, value } = filter; @@ -522,39 +541,31 @@ const Filters = ({ ))} - {areMoreThanOneFilters ? ( - <> - - - - handleDeleteFilterButtonClick(filterIndex) - } - size="large" - > - - delete_outline - - - - - - - - - - - ) : null} + <> + + + handleDeleteFilterButtonClick(filterIndex)} + size="large" + > + delete_outline + + + + + + + + + ); @@ -599,7 +610,7 @@ const Filters = ({ startIcon={search} onClick={handleApplyButtonClick} disabled={ - handleApplyValidation(filterParameters, filtersConfig) != null + handleApplyValidation(filterParameters, filtersConfig) !== null } > Search diff --git a/moped-editor/src/components/GridTable/helpers.js b/moped-editor/src/components/GridTable/helpers.js index a6dfa34c91..a854533089 100644 --- a/moped-editor/src/components/GridTable/helpers.js +++ b/moped-editor/src/components/GridTable/helpers.js @@ -96,7 +96,7 @@ export const handleApplyValidation = (filterParameters, filtersConfig) => { if (filterParameters) { if (filterParameters.length === 0) { - feedback.push("• No filters have been added."); + // feedback.push("• No filters have been added."); } else { filterParameters.forEach((filter) => { const { field: fieldName, value, operator } = filter; From 6896bc628ce320fb435108b5456e5e7db5ea3c60 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 6 Feb 2024 17:12:05 -0600 Subject: [PATCH 20/76] Remove validation block that handles 0 filters; update delete handler comments --- .../src/components/GridTable/Filters.js | 39 ++++++++-------- .../src/components/GridTable/helpers.js | 44 +++++++++---------- 2 files changed, 40 insertions(+), 43 deletions(-) diff --git a/moped-editor/src/components/GridTable/Filters.js b/moped-editor/src/components/GridTable/Filters.js index 95242d49d2..cb301bc726 100644 --- a/moped-editor/src/components/GridTable/Filters.js +++ b/moped-editor/src/components/GridTable/Filters.js @@ -228,14 +228,15 @@ const Filters = ({ * @param {string} filterIndex - The index of the filter to be deleted */ const handleDeleteFilterButtonClick = (filterIndex) => { - // Clone the state, delete the filter index of the button clicked + /* Clone the state, delete the filter index of the button clicked, and update filter state */ const filtersNewState = [...filterParameters]; filtersNewState.splice(filterIndex, 1); + setFilterParameters(filtersNewState); const remainingFiltersCount = filtersNewState.length; if (remainingFiltersCount === 0) { - setFilterParameters([]); + /* Clear search params since we have no advanced filters */ setSearchParams((prevSearchParams) => { prevSearchParams.delete(advancedSearchFilterParamName); prevSearchParams.delete(advancedSearchIsOrParamName); @@ -244,11 +245,9 @@ const Filters = ({ }); } else if (remainingFiltersCount === 1) { /* Reset isOr to false (all/and) if there is only one filter left */ - setFilterParameters(filtersNewState); setIsOrToggleValue(false); } else { - // Finally, reset the state - setFilterParameters(filtersNewState); + /* Remove the details of the removed filter from search params */ const jsonParamString = JSON.stringify(filtersNewState); setSearchParams((prevSearchParams) => { prevSearchParams.set(advancedSearchFilterParamName, jsonParamString); @@ -293,16 +292,20 @@ const Filters = ({ * Applies the current local state and updates the parent's state */ const handleApplyButtonClick = () => { - setSearchParams((prevSearchParams) => { - const jsonParamString = JSON.stringify(filterParameters); + /* If we have advanced filters, set query state values and update search params */ + if (filterParameters.length > 0) { + setSearchParams((prevSearchParams) => { + const jsonParamString = JSON.stringify(filterParameters); - prevSearchParams.set(advancedSearchFilterParamName, jsonParamString); - prevSearchParams.set(advancedSearchIsOrParamName, isOrToggleValue); - return prevSearchParams; - }); + prevSearchParams.set(advancedSearchFilterParamName, jsonParamString); + prevSearchParams.set(advancedSearchIsOrParamName, isOrToggleValue); + return prevSearchParams; + }); + + setIsOr(isOrToggleValue); + setFilters(filterParameters); + } - setIsOr(isOrToggleValue); - setFilters(filterParameters); handleAdvancedSearchClose(); }; @@ -365,13 +368,11 @@ const Filters = ({ {filterParameters.length === 0 ? ( - - - - No filters applied - + + + No filters applied - + ) : null} {filterParameters.map((filter, filterIndex) => { const { field: fieldName, operator, value } = filter; diff --git a/moped-editor/src/components/GridTable/helpers.js b/moped-editor/src/components/GridTable/helpers.js index a854533089..47f634e818 100644 --- a/moped-editor/src/components/GridTable/helpers.js +++ b/moped-editor/src/components/GridTable/helpers.js @@ -95,34 +95,30 @@ export const handleApplyValidation = (filterParameters, filtersConfig) => { let feedback = []; if (filterParameters) { - if (filterParameters.length === 0) { - // feedback.push("• No filters have been added."); - } else { - filterParameters.forEach((filter) => { - const { field: fieldName, value, operator } = filter; - const fieldConfig = filtersConfig.fields.find( - (field) => field.name === fieldName - ); - const type = fieldConfig?.type; + filterParameters.forEach((filter) => { + const { field: fieldName, value, operator } = filter; + const fieldConfig = filtersConfig.fields.find( + (field) => field.name === fieldName + ); + const type = fieldConfig?.type; - if (fieldName === null) { - feedback.push("• One or more fields have not been selected."); - } + if (fieldName === null) { + feedback.push("• One or more fields have not been selected."); + } - if (operator === null) { - feedback.push("• One or more operators have not been selected."); - } + if (operator === null) { + feedback.push("• One or more operators have not been selected."); + } - if (value === null || value.trim() === "") { - if (operator && !isFilterNullType(operator)) { - feedback.push("• One or more missing values."); - } - } - if (!checkIsValidInput(filter, type)) { - feedback.push("• One or more invalid inputs."); + if (value === null || value.trim() === "") { + if (operator && !isFilterNullType(operator)) { + feedback.push("• One or more missing values."); } - }); - } + } + if (!checkIsValidInput(filter, type)) { + feedback.push("• One or more invalid inputs."); + } + }); } return feedback.length > 0 ? feedback : null; }; From 61bb50dff4efc57b68d0d862587d47705b427e0f Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 6 Feb 2024 17:13:55 -0600 Subject: [PATCH 21/76] Handle removing last adv filter and retaining search box filter --- moped-editor/src/components/GridTable/Filters.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/moped-editor/src/components/GridTable/Filters.js b/moped-editor/src/components/GridTable/Filters.js index cb301bc726..70d84038fd 100644 --- a/moped-editor/src/components/GridTable/Filters.js +++ b/moped-editor/src/components/GridTable/Filters.js @@ -292,8 +292,8 @@ const Filters = ({ * Applies the current local state and updates the parent's state */ const handleApplyButtonClick = () => { - /* If we have advanced filters, set query state values and update search params */ if (filterParameters.length > 0) { + /* If we have advanced filters, set query state values and update search params */ setSearchParams((prevSearchParams) => { const jsonParamString = JSON.stringify(filterParameters); @@ -304,6 +304,10 @@ const Filters = ({ setIsOr(isOrToggleValue); setFilters(filterParameters); + } else { + /* If we have no advanced filters, reset query state */ + setFilters([]); + setIsOr(false); } handleAdvancedSearchClose(); From f6db79a676b349f9288a14a5dcea2dfb9ff8a0ac Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Tue, 6 Feb 2024 18:11:14 -0600 Subject: [PATCH 22/76] add new columns --- moped-database/metadata/tables.yaml | 33 +++++++++++-------- .../down.sql | 3 ++ .../up.sql | 6 ++++ 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index 650a26fcd4..fd8a8af44d 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -2643,7 +2643,6 @@ updated_by_user_id: x-hasura-user-db-id columns: - completed - - created_at - milestone_id - description - date_actual @@ -2660,7 +2659,6 @@ updated_by_user_id: x-hasura-user-db-id columns: - completed - - created_at - milestone_id - description - date_actual @@ -2675,49 +2673,57 @@ columns: - completed - created_at - - milestone_id - - description + - created_by_user_id - date_actual - date_estimate + - description + - is_deleted + - milestone_id - milestone_order - project_id - project_milestone_id - - is_deleted + - updated_at + - updated_by_user_id filter: {} - role: moped-editor permission: columns: - completed - created_at - - milestone_id - - description + - created_by_user_id - date_actual - date_estimate + - description + - is_deleted + - milestone_id - milestone_order - project_id - project_milestone_id - - is_deleted + - updated_at + - updated_by_user_id filter: {} - role: moped-viewer permission: columns: - completed - created_at - - milestone_id - - description + - created_by_user_id - date_actual - date_estimate + - description + - is_deleted + - milestone_id - milestone_order - project_id - project_milestone_id - - is_deleted + - updated_at + - updated_by_user_id filter: {} update_permissions: - role: moped-admin permission: columns: - completed - - created_at - milestone_id - description - date_actual @@ -2727,11 +2733,11 @@ - is_deleted filter: {} check: {} + comment: No update permissions on audit fields - role: moped-editor permission: columns: - completed - - created_at - milestone_id - description - date_actual @@ -2741,6 +2747,7 @@ - is_deleted filter: {} check: {} + comment: No update permissions on audit fields event_triggers: - name: activity_log_moped_proj_milestones definition: diff --git a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql index 16758402c6..f6fd8ce2cc 100644 --- a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql +++ b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql @@ -1 +1,4 @@ alter table "public"."moped_proj_milestones" rename column "created_at" to "date_added"; +alter table "public"."moped_proj_milestones" drop column "created_by_user_id"; +alter table "public"."moped_proj_milestones" drop column "updated_at"; +alter table "public"."moped_proj_milestones" drop column "updated_by_user_id"; diff --git a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql index f1334fbebf..65b3300bce 100644 --- a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql +++ b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql @@ -1 +1,7 @@ alter table "public"."moped_proj_milestones" rename column "date_added" to "created_at"; +alter table "public"."moped_proj_milestones" add column "created_by_user_id" integer + null; +alter table "public"."moped_proj_milestones" add column "updated_at" timestamptz + null; +alter table "public"."moped_proj_milestones" add column "updated_by_user_id" integer + null; From a110825410fe87f688affb5498af2f5f6f2f4421 Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Tue, 6 Feb 2024 18:17:12 -0600 Subject: [PATCH 23/76] fix update permissions --- moped-database/metadata/tables.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index fd8a8af44d..b7ac673c6f 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -2722,6 +2722,8 @@ update_permissions: - role: moped-admin permission: + set: + updated_by_user_id: x-hasura-user-db-id columns: - completed - milestone_id @@ -2736,6 +2738,8 @@ comment: No update permissions on audit fields - role: moped-editor permission: + set: + updated_by_user_id: x-hasura-user-db-id columns: - completed - milestone_id From 2682c30e46dac3750d39f6e911ab57d2bdd95fad Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Tue, 6 Feb 2024 18:20:51 -0600 Subject: [PATCH 24/76] reorder set --- moped-database/metadata/tables.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index b7ac673c6f..6025f3a4f8 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -2722,8 +2722,6 @@ update_permissions: - role: moped-admin permission: - set: - updated_by_user_id: x-hasura-user-db-id columns: - completed - milestone_id @@ -2735,11 +2733,11 @@ - is_deleted filter: {} check: {} + set: + updated_by_user_id: x-hasura-user-db-id comment: No update permissions on audit fields - role: moped-editor permission: - set: - updated_by_user_id: x-hasura-user-db-id columns: - completed - milestone_id @@ -2751,6 +2749,8 @@ - is_deleted filter: {} check: {} + set: + updated_by_user_id: x-hasura-user-db-id comment: No update permissions on audit fields event_triggers: - name: activity_log_moped_proj_milestones From 56f852211ec0cddfc94b88cb0fef8740a4f69ac0 Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Tue, 6 Feb 2024 18:59:35 -0600 Subject: [PATCH 25/76] undo changes --- moped-database/metadata/tables.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index 6025f3a4f8..662621831c 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -3432,6 +3432,8 @@ update_permissions: - role: moped-admin permission: + set: + updated_by_user_id: x-hasura-user-db-id columns: - is_current_phase - is_deleted @@ -3445,8 +3447,6 @@ - is_phase_end_confirmed filter: {} check: {} - set: - updated_by_user_id: x-hasura-user-db-id - role: moped-editor permission: columns: @@ -3462,8 +3462,6 @@ - is_phase_end_confirmed filter: {} check: {} - set: - updated_by_user_id: x-hasura-user-db-id event_triggers: - name: activity_log_moped_proj_phases definition: From 5ac052354a7d7fec0018060bdf95d7bb6ecf6a2e Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Tue, 6 Feb 2024 19:00:20 -0600 Subject: [PATCH 26/76] redo --- moped-database/metadata/tables.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index 662621831c..c70ea9da7f 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -3462,6 +3462,8 @@ - is_phase_end_confirmed filter: {} check: {} + set: + updated_by_user_id: x-hasura-user-db-id event_triggers: - name: activity_log_moped_proj_phases definition: From a601c3e8cb3f8c4728262c7a5fcab50a0d3b03bf Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Tue, 6 Feb 2024 19:01:28 -0600 Subject: [PATCH 27/76] redo undo redo --- moped-database/metadata/tables.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index c70ea9da7f..237df705c9 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -3449,6 +3449,8 @@ check: {} - role: moped-editor permission: + set: + updated_by_user_id: x-hasura-user-db-id columns: - is_current_phase - is_deleted @@ -3462,8 +3464,6 @@ - is_phase_end_confirmed filter: {} check: {} - set: - updated_by_user_id: x-hasura-user-db-id event_triggers: - name: activity_log_moped_proj_phases definition: From 440dac8c9afc95a5da643a30c339bae1cc0e231b Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 7 Feb 2024 11:48:50 -0600 Subject: [PATCH 28/76] proposed sqlfluff settings --- .sqlfluff | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .sqlfluff diff --git a/.sqlfluff b/.sqlfluff new file mode 100644 index 0000000000..ef4951a446 --- /dev/null +++ b/.sqlfluff @@ -0,0 +1,71 @@ +[sqlfluff] + +# Supported dialects https://docs.sqlfluff.com/en/stable/dialects.html +# Or run 'sqlfluff dialects' +dialect = postgres + +large_file_skip_char_limit = 0 +large_file_skip_byte_limit = 0 + +# One of [raw|jinja|python|placeholder] +templater = jinja + +# Comma separated list of rules to exclude, or None +# See https://docs.sqlfluff.com/en/stable/configuration.html#enabling-and-disabling-rules +# AM04 (ambiguous.column_count) and ST06 (structure.column_order) are +# two of the more controversial rules included to illustrate usage. +exclude_rules = ambiguous.column_count, structure.column_order + +# The standard max_line_length is 80 in line with the convention of +# other tools and several style guides. Many projects however prefer +# something a little longer. +# Set to zero or negative to disable checks. +max_line_length = 120 + +# CPU processes to use while linting. +# The default is "single threaded" to allow easy debugging, but this +# is often undesirable at scale. +# If positive, just implies number of processes. +# If negative or zero, implies number_of_cpus - specified_number. +# e.g. -1 means use all processors but one. 0 means all cpus. +processes = -1 + +# If using the dbt templater, we recommend setting the project dir. +# [sqlfluff:templater:dbt] +# project_dir = ./ + +[sqlfluff:indentation] +# While implicit indents are not enabled by default. Many of the +# SQLFluff maintainers do use them in their projects. +allow_implicit_indents = True + +# The default configuration for aliasing rules is "consistent" +# which will auto-detect the setting from the rest of the file. This +# is less desirable in a new project and you may find this (slightly +# more strict) setting more useful. +# [sqlfluff:rules:aliasing.table] +# aliasing = explicit +# [sqlfluff:rules:aliasing.column] +# aliasing = explicit +# [sqlfluff:rules:aliasing.length] +# min_alias_length = 3 + +# The default configuration for capitalisation rules is "consistent" +# which will auto-detect the setting from the rest of the file. This +# is less desirable in a new project and you may find this (slightly +# more strict) setting more useful. +# Typically we find users rely on syntax highlighting rather than +# capitalisation to distinguish between keywords and identifiers. +# Clearly, if your organisation has already settled on uppercase +# formatting for any of these syntax elements then set them to "upper". +# See https://stackoverflow.com/questions/608196/why-should-i-capitalize-my-sql-keywords-is-there-a-good-reason +[sqlfluff:rules:capitalisation.keywords] +capitalisation_policy = upper +[sqlfluff:rules:capitalisation.identifiers] +capitalisation_policy = upper +[sqlfluff:rules:capitalisation.functions] +extended_capitalisation_policy = consistent +[sqlfluff:rules:capitalisation.literals] +capitalisation_policy = consistent +[sqlfluff:rules:capitalisation.types] +extended_capitalisation_policy = consistent \ No newline at end of file From ac4170df0cddfafcb99176d7e8d1039f3d7cba28 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 7 Feb 2024 12:28:17 -0600 Subject: [PATCH 29/76] Lint the files with sqlfluff --- .../views/component_arcgis_online_view.sql | 1258 +++++++++-------- moped-database/views/current_phase_view.sql | 18 +- moped-database/views/project_geography.sql | 23 +- moped-database/views/project_list_view.sql | 396 +++--- moped-database/views/uniform_features.sql | 122 +- 5 files changed, 979 insertions(+), 838 deletions(-) diff --git a/moped-database/views/component_arcgis_online_view.sql b/moped-database/views/component_arcgis_online_view.sql index b9af383dbc..8a5e6d1896 100644 --- a/moped-database/views/component_arcgis_online_view.sql +++ b/moped-database/views/component_arcgis_online_view.sql @@ -1,76 +1,215 @@ -- current_version: 1706897389736_fix_sub_comp_date_dash CREATE OR REPLACE VIEW component_arcgis_online_view AS ( - SELECT - mpc.project_id, - comp_geography.project_component_id, - comp_geography.feature_ids, - mpc.component_id, - comp_geography.geometry, + WITH work_types AS ( + -- group work types by project component id + SELECT + project_component_id, + string_agg(mwt.name, ', ') AS work_types + FROM + moped_proj_component_work_types AS mpcwt + LEFT JOIN moped_work_types AS mwt ON mpcwt.work_type_id = mwt.id + WHERE mpcwt.is_deleted = FALSE + GROUP BY + project_component_id + ), +council_districts AS ( + -- group council districts by project component id + SELECT + component_id AS project_component_id, + string_agg( + DISTINCT council_district_id::text, + ', ' + ) AS council_districts + FROM + features_council_districts + LEFT JOIN features ON features_council_districts.feature_id = features.id + WHERE + features.is_deleted = FALSE + GROUP BY + component_id + ), +comp_geography AS ( + -- group feature properties by project component ID + SELECT + component_id AS project_component_id, + string_agg(DISTINCT id::text, ', ') AS feature_ids, + st_asgeojson( + st_union( + array_agg(geography) + ) + )::json AS "geometry", + st_asgeojson( + st_union( + array_agg(line_geography) + ) + )::json AS "line_geometry", + string_agg(DISTINCT signal_id::text, ', ') AS signal_ids, + sum(length_feet) AS length_feet_total + FROM + ( + -- union all features + SELECT + id, + feature_signals.component_id, + feature_signals.geography::geometry, + st_exteriorring( + st_buffer(feature_signals.geography, 7)::geometry + ) AS line_geography, + feature_signals.signal_id, + NULL AS length_feet + FROM + feature_signals + WHERE + feature_signals.is_deleted = FALSE + UNION ALL + SELECT + id, + feature_street_segments.component_id, + feature_street_segments.geography::geometry, + feature_street_segments.geography::geometry AS line_geography, + NULL AS signal_id, + length_feet + FROM + feature_street_segments + WHERE + feature_street_segments.is_deleted = FALSE + UNION ALL + SELECT + id, + feature_intersections.component_id, + feature_intersections.geography::geometry, + st_exteriorring( + st_buffer(feature_intersections.geography, 7)::geometry + ) AS line_geography, + NULL AS signal_id, + NULL AS length_feet + FROM + feature_intersections + WHERE + feature_intersections.is_deleted = FALSE + UNION ALL + SELECT + id, + feature_drawn_points.component_id, + feature_drawn_points.geography::geometry, + st_exteriorring( + st_buffer(feature_drawn_points.geography, 7)::geometry + ) AS line_geography, + NULL AS signal_id, + NULL AS length_feet + FROM + feature_drawn_points + WHERE + feature_drawn_points.is_deleted = FALSE + UNION ALL + SELECT + id, + feature_drawn_lines.component_id, + feature_drawn_lines.geography::geometry, + feature_drawn_lines.geography::geometry AS line_geography, + NULL AS signal_id, + length_feet + FROM + feature_drawn_lines + WHERE + feature_drawn_lines.is_deleted = FALSE + ) AS feature_union + GROUP BY + component_id + ), +subcomponents AS ( + -- group subcomponents by project component id + SELECT + project_component_id, + string_agg(ms.subcomponent_name, ', ') AS subcomponents + FROM + moped_proj_components_subcomponents AS mpcs + LEFT JOIN moped_subcomponents AS ms ON mpcs.subcomponent_id = ms.subcomponent_id + WHERE mpcs.is_deleted = FALSE + GROUP BY + project_component_id + ), +component_tags AS ( + -- group project component tags by project component id + SELECT + project_component_id, + string_agg(mct.type || ' - ' || mct.name, ', ') AS component_tags + FROM + moped_proj_component_tags AS mpct + LEFT JOIN moped_component_tags AS mct ON mpct.component_tag_id = mct.id + WHERE mpct.is_deleted = FALSE + GROUP BY + project_component_id + ) + SELECT + mpc.project_id, + comp_geography.project_component_id, + comp_geography.feature_ids, + mpc.component_id, + comp_geography.geometry, comp_geography.line_geometry, - comp_geography.signal_ids, - council_districts.council_districts, - 'placeholder text' as council_districts_searchable, - CASE WHEN council_districts IS NULL OR council_districts = '' THEN FALSE - ELSE TRUE - END AS is_within_city_limits, - comp_geography.length_feet_total, - round((comp_geography.length_feet_total / 5280::numeric),2) AS length_miles_total, - mc.component_name, - mc.component_subtype, - mc.component_name_full, - 'placeholder text' as component_categories, - subcomponents.subcomponents as component_subcomponents, - work_types.work_types as component_work_types, - component_tags.component_tags, + comp_geography.signal_ids, + council_districts.council_districts, + 'placeholder text' AS council_districts_searchable, + NOT coalesce (council_districts IS NULL OR council_districts = '', FALSE) AS is_within_city_limits, + comp_geography.length_feet_total, + round((comp_geography.length_feet_total / 5280::numeric), 2) AS length_miles_total, + mc.component_name, + mc.component_subtype, + mc.component_name_full, + 'placeholder text' AS component_categories, + subcomponents.subcomponents AS component_subcomponents, + work_types.work_types AS component_work_types, + component_tags.component_tags, mpc.description AS component_description, - mpc.interim_project_component_id, + mpc.interim_project_component_id, mpc.completion_date, - COALESCE(mpc.completion_date, substantial_completion_date) as substantial_completion_date, - '2024-01-01T00:00:00-06:00' as substantial_completion_date_estimated, - mpc.srts_id, - mpc.location_description as component_location_description, + coalesce(mpc.completion_date, substantial_completion_date) AS substantial_completion_date, + '2024-01-01T00:00:00-06:00' AS substantial_completion_date_estimated, + mpc.srts_id, + mpc.location_description AS component_location_description, plv.project_name, - 'placeholder text' as project_name_descriptor, - 'placeholder text' as project_name_with_descriptor, - plv.project_description, - plv.ecapris_subproject_id, + 'placeholder text' AS project_name_descriptor, + 'placeholder text' AS project_name_with_descriptor, + plv.project_description, + plv.ecapris_subproject_id, plv.project_website, - plv.updated_at as project_updated_at, - mpc.phase_id AS component_phase_id, - mph.phase_name AS component_phase_name, - mph.phase_name_simple as component_phase_name_simple, - current_phase.phase_id AS project_phase_id, - current_phase.phase_name AS project_phase_name, - current_phase.phase_name_simple AS project_phase_name_simple, - COALESCE(mph.phase_name, current_phase.phase_name) AS current_phase_name, - COALESCE(mph.phase_name_simple, current_phase.phase_name_simple) AS current_phase_name_simple, - plv.project_team_members, - plv.project_sponsor, - plv.project_lead, - plv.public_process_status, - plv.interim_project_id, - plv.project_partners, - plv.task_order_names, - plv.funding_source_name, - plv.type_name, - plv.project_status_update, - plv.project_status_update_date_created, - plv.construction_start_date, - plv.completion_end_date, - plv.project_inspector, - plv.project_designer, - plv.project_tags, - plv.workgroup_contractors, - plv.contract_numbers, + plv.updated_at AS project_updated_at, + mpc.phase_id AS component_phase_id, + mph.phase_name AS component_phase_name, + mph.phase_name_simple AS component_phase_name_simple, + current_phase.phase_id AS project_phase_id, + current_phase.phase_name AS project_phase_name, + current_phase.phase_name_simple AS project_phase_name_simple, + coalesce(mph.phase_name, current_phase.phase_name) AS current_phase_name, + coalesce(mph.phase_name_simple, current_phase.phase_name_simple) AS current_phase_name_simple, + plv.project_team_members, + plv.project_sponsor, + plv.project_lead, + plv.public_process_status, + plv.interim_project_id, + plv.project_partners, + plv.task_order_names, + plv.funding_source_name, + plv.type_name, + plv.project_status_update, + plv.project_status_update_date_created, + plv.construction_start_date, + plv.completion_end_date, + plv.project_inspector, + plv.project_designer, + plv.project_tags, + plv.workgroup_contractors, + plv.contract_numbers, plv.parent_project_id, plv.parent_project_name, plv.parent_project_url, - 'placeholder text' as parent_project_name_with_descriptor, - 'placeholder text' as related_project_ids, - 'placeholder text' as related_project_ids_searchable, - plv.knack_project_id as knack_data_tracker_project_record_id, + 'placeholder text' AS parent_project_name_with_descriptor, + 'placeholder text' AS related_project_ids, + 'placeholder text' AS related_project_ids_searchable, + plv.knack_project_id AS knack_data_tracker_project_record_id, plv.project_url, - plv.project_url || '?tab=map&project_component_id=' || mpc.project_component_id :: text as component_url, + plv.project_url || '?tab=map&project_component_id=' || mpc.project_component_id::text AS component_url, plv.project_development_status, plv.project_development_status_date, plv.project_development_status_date_calendar_year, @@ -79,157 +218,21 @@ CREATE OR REPLACE VIEW component_arcgis_online_view AS ( plv.project_development_status_date_calendar_year_quarter, plv.project_development_status_date_fiscal_year, plv.project_development_status_date_fiscal_year_quarter, - added_by as project_added_by - FROM - moped_proj_components mpc - LEFT JOIN ( - -- group feature properties by project component ID - SELECT - component_id AS project_component_id, - STRING_AGG(DISTINCT id :: text, ', ') AS feature_ids, - ST_AsGeoJSON( - ST_Union( - ARRAY_AGG(geography) - ) - ):: json AS "geometry", - ST_AsGeoJSON( - ST_Union( - ARRAY_AGG(line_geography) - ) - ):: json AS "line_geometry", - STRING_AGG(DISTINCT signal_id :: text, ', ') AS signal_ids, - SUM(length_feet) as length_feet_total - FROM - ( - -- union all features - SELECT - id, - feature_signals.component_id, - feature_signals.geography :: geometry, - ST_ExteriorRing( - ST_Buffer(feature_signals.geography, 7):: geometry - ) AS line_geography, - feature_signals.signal_id, - NULL AS length_feet - FROM - feature_signals - WHERE - feature_signals.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_street_segments.component_id, - feature_street_segments.geography :: geometry, - feature_street_segments.geography :: geometry as line_geography, - NULL AS signal_id, - length_feet - FROM - feature_street_segments - WHERE - feature_street_segments.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_intersections.component_id, - feature_intersections.geography :: geometry, - ST_ExteriorRing( - ST_Buffer(feature_intersections.geography, 7):: geometry - ) AS line_geography, - NULL AS signal_id, - NULL AS length_feet - FROM - feature_intersections - WHERE - feature_intersections.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_drawn_points.component_id, - feature_drawn_points.geography :: geometry, - ST_ExteriorRing( - ST_Buffer(feature_drawn_points.geography, 7):: geometry - ) AS line_geography, - NULL AS signal_id, - NULL AS length_feet - FROM - feature_drawn_points - WHERE - feature_drawn_points.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_drawn_lines.component_id, - feature_drawn_lines.geography :: geometry, - feature_drawn_lines.geography :: geometry as line_geography, - NULL AS signal_id, - length_feet - FROM - feature_drawn_lines - WHERE - feature_drawn_lines.is_deleted = FALSE - ) feature_union - GROUP BY - component_id - ) comp_geography ON comp_geography.project_component_id = mpc.project_component_id - LEFT JOIN ( - -- group council districts by project component id - SELECT - component_id AS project_component_id, - STRING_AGG( - DISTINCT council_district_id :: text, - ', ' - ) AS council_districts - FROM - features_council_districts - LEFT JOIN features ON features.id = features_council_districts.feature_id - WHERE - features.is_deleted = FALSE - GROUP BY - component_id - ) council_districts ON council_districts.project_component_id = mpc.project_component_id - LEFT JOIN ( - -- group subcomponents by project component id - SELECT - project_component_id, - string_agg(ms.subcomponent_name, ', ') subcomponents - FROM - moped_proj_components_subcomponents mpcs - LEFT JOIN moped_subcomponents ms ON mpcs.subcomponent_id = ms.subcomponent_id - WHERE mpcs.is_deleted = FALSE - GROUP BY - project_component_id - ) subcomponents ON subcomponents.project_component_id = mpc.project_component_id - LEFT JOIN ( - -- group work types by project component id - SELECT - project_component_id, - string_agg(mwt.name, ', ') work_types - FROM - moped_proj_component_work_types mpcwt - LEFT JOIN moped_work_types mwt ON mpcwt.work_type_id = mwt.id - WHERE mpcwt.is_deleted = FALSE - GROUP BY - project_component_id - ) work_types ON work_types.project_component_id = mpc.project_component_id - LEFT JOIN ( - -- group project component tags by project component id - SELECT - project_component_id, - string_agg(mct.type || ' - ' || mct.name, ', ') component_tags - FROM - moped_proj_component_tags mpct - LEFT JOIN moped_component_tags mct ON mpct.component_tag_id = mct.id - WHERE mpct.is_deleted = FALSE - GROUP BY - project_component_id - ) component_tags ON component_tags.project_component_id = mpc.project_component_id - LEFT JOIN project_list_view plv ON plv.project_id = mpc.project_id - LEFT JOIN current_phase_view current_phase ON mpc.project_id = current_phase.project_id - LEFT JOIN moped_phases mph ON mpc.phase_id = mph.phase_id - LEFT JOIN moped_components mc ON mc.component_id = mpc.component_id -WHERE - mpc.is_deleted = FALSE - AND plv.is_deleted = FALSE + added_by AS project_added_by + FROM + moped_proj_components AS mpc + LEFT JOIN comp_geography ON mpc.project_component_id = comp_geography.project_component_id + LEFT JOIN council_districts ON mpc.project_component_id = council_districts.project_component_id + LEFT JOIN subcomponents ON mpc.project_component_id = subcomponents.project_component_id + LEFT JOIN work_types ON mpc.project_component_id = work_types.project_component_id + LEFT JOIN component_tags ON mpc.project_component_id = component_tags.project_component_id + LEFT JOIN project_list_view AS plv ON mpc.project_id = plv.project_id + LEFT JOIN current_phase_view AS current_phase ON mpc.project_id = current_phase.project_id + LEFT JOIN moped_phases AS mph ON mpc.phase_id = mph.phase_id + LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id + WHERE + mpc.is_deleted = FALSE + AND plv.is_deleted = FALSE ); DROP VIEW project_list_view CASCADE; @@ -237,63 +240,89 @@ DROP VIEW project_list_view CASCADE; CREATE OR REPLACE VIEW public.project_list_view AS WITH project_person_list_lookup AS ( SELECT - mpp.project_id, - string_agg(DISTINCT concat(mu.first_name, ' ', mu.last_name, ':', mpr.project_role_name), ','::text) AS project_team_members - FROM moped_proj_personnel mpp - JOIN moped_users mu ON mpp.user_id = mu.user_id - JOIN moped_proj_personnel_roles mppr ON mpp.project_personnel_id = mppr.project_personnel_id - JOIN moped_project_roles mpr ON mppr.project_role_id = mpr.project_role_id - WHERE mpp.is_deleted = false - AND mppr.is_deleted = false + mpp.project_id, + string_agg( + DISTINCT concat(mu.first_name, ' ', mu.last_name, ':', mpr.project_role_name), ','::text + ) AS project_team_members + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS mu ON mpp.user_id = mu.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE mpp.is_deleted = FALSE + AND mppr.is_deleted = FALSE GROUP BY mpp.project_id - ), funding_sources_lookup AS ( - SELECT - mpf_1.project_id, - string_agg(mfs.funding_source_name, ', '::text) AS funding_source_name - FROM moped_proj_funding mpf_1 - LEFT JOIN moped_fund_sources mfs ON mpf_1.funding_source_id = mfs.funding_source_id - WHERE mpf_1.is_deleted = false +), + +funding_sources_lookup AS ( + SELECT + mpf_1.project_id, + string_agg(mfs.funding_source_name, ', '::text) AS funding_source_name + FROM moped_proj_funding AS mpf_1 + LEFT JOIN moped_fund_sources AS mfs ON mpf_1.funding_source_id = mfs.funding_source_id + WHERE mpf_1.is_deleted = FALSE GROUP BY mpf_1.project_id - ), project_type_lookup AS ( +), + +project_type_lookup AS ( SELECT - mpt.project_id, - string_agg(mt.type_name, ', '::text) AS type_name - FROM moped_project_types mpt - LEFT JOIN moped_types mt ON mpt.project_type_id = mt.type_id AND mpt.is_deleted = false + mpt.project_id, + string_agg(mt.type_name, ', '::text) AS type_name + FROM moped_project_types AS mpt + LEFT JOIN moped_types AS mt ON mpt.project_type_id = mt.type_id AND mpt.is_deleted = FALSE GROUP BY mpt.project_id - ), child_project_lookup AS ( - SELECT jsonb_agg(children.project_id) AS children_project_ids, - children.parent_project_id AS parent_id - FROM moped_project AS children - JOIN moped_project AS parent ON (parent.project_id = children.parent_project_id) - WHERE children.is_deleted = false +), + +child_project_lookup AS ( + SELECT + jsonb_agg(children.project_id) AS children_project_ids, + children.parent_project_id AS parent_id + FROM moped_project AS children + INNER JOIN moped_project AS parent ON (children.parent_project_id = parent.project_id) + WHERE children.is_deleted = FALSE GROUP BY parent_id - ), work_activities AS ( - SELECT - project_id, - string_agg(task_order_objects.task_order_object ->> 'display_name'::text, - ', '::text) AS task_order_names, - string_agg(task_order_objects.task_order_object ->> 'task_order'::text, - ', '::text) AS task_order_names_short, - jsonb_agg(DISTINCT task_order_objects.task_order_object) FILTER (WHERE task_order_objects.task_order_object IS NOT NULL) AS task_orders, - string_agg(DISTINCT mpwa.workgroup_contractor, - ', '::text) AS workgroup_contractors, - string_agg(mpwa.contract_number, - ', '::text) AS contract_numbers FROM moped_proj_work_activity mpwa - LEFT JOIN LATERAL jsonb_array_elements(mpwa.task_orders) task_order_objects (task_order_object) ON TRUE WHERE 1 = 1 - AND mpwa.is_deleted = FALSE - GROUP BY - mpwa.project_id - ), moped_proj_components_subtypes AS ( +), + +work_activities AS ( + SELECT + project_id, + string_agg( + task_order_objects.task_order_object ->> 'display_name'::text, + ', '::text + ) AS task_order_names, + string_agg( + task_order_objects.task_order_object ->> 'task_order'::text, + ', '::text + ) AS task_order_names_short, + jsonb_agg(DISTINCT task_order_objects.task_order_object) FILTER ( + WHERE task_order_objects.task_order_object IS NOT NULL + ) AS task_orders, + string_agg( + DISTINCT mpwa.workgroup_contractor, + ', '::text + ) AS workgroup_contractors, + string_agg( + mpwa.contract_number, + ', '::text + ) AS contract_numbers + FROM moped_proj_work_activity AS mpwa + LEFT JOIN LATERAL jsonb_array_elements(mpwa.task_orders) AS task_order_objects (task_order_object) + ON TRUE WHERE 1 = 1 + AND mpwa.is_deleted = FALSE + GROUP BY + mpwa.project_id +), + +moped_proj_components_subtypes AS ( SELECT - mpc.project_id, - string_agg(DISTINCT mc.component_name_full, ', '::text) AS components - FROM moped_proj_components mpc - LEFT JOIN moped_components mc ON mpc.component_id = mc.component_id + mpc.project_id, + string_agg(DISTINCT mc.component_name_full, ', '::text) AS components + FROM moped_proj_components AS mpc + LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id WHERE mpc.is_deleted = FALSE GROUP BY mpc.project_id - ) - SELECT +) + +SELECT mp.project_id, mp.project_name, mp.project_description, @@ -302,9 +331,9 @@ AS WITH project_person_list_lookup AS ( mp.date_added, mp.is_deleted, mp.updated_at, - current_phase.phase_name as current_phase, - current_phase.phase_key as current_phase_key, - current_phase.phase_name_simple as current_phase_simple, + current_phase.phase_name AS current_phase, + current_phase.phase_key AS current_phase_key, + current_phase.phase_name_simple AS current_phase_simple, ppll.project_team_members, me.entity_name AS project_sponsor, mel.entity_name AS project_lead, @@ -312,154 +341,172 @@ AS WITH project_person_list_lookup AS ( mp.interim_project_id, mp.parent_project_id, mp.knack_project_id, - 'https://mobility.austin.gov/moped/projects/' || mp.project_id :: text as project_url, - 'https://mobility.austin.gov/moped/projects/' || mp.parent_project_id :: text as parent_project_url, - proj_status_update.project_note as project_status_update, - proj_status_update.date_created as project_status_update_date_created, + 'https://mobility.austin.gov/moped/projects/' || mp.project_id::text AS project_url, + 'https://mobility.austin.gov/moped/projects/' || mp.parent_project_id::text AS parent_project_url, + proj_status_update.project_note AS project_status_update, + proj_status_update.date_created AS project_status_update_date_created, work_activities.workgroup_contractors, work_activities.contract_numbers, work_activities.task_order_names, work_activities.task_order_names_short, work_activities.task_orders, - 'placeholder text' as project_development_status, - '2024-01-01T00:00:00-06:00' as project_development_status_date, - 9999 as project_development_status_date_calendar_year, - 'placeholder text' as project_development_status_date_calendar_year_month, - 'placeholder text' as project_development_status_date_calendar_year_month_numeric, - 'placeholder text' as project_development_status_date_calendar_year_quarter, - 999 as project_development_status_date_fiscal_year, - 'placeholder text' as project_development_status_date_fiscal_year_quarter, - (SELECT project_name - FROM moped_project - WHERE project_id = mp.parent_project_id - ) as parent_project_name, + 'placeholder text' AS project_development_status, + '2024-01-01T00:00:00-06:00' AS project_development_status_date, + 9999 AS project_development_status_date_calendar_year, + 'placeholder text' AS project_development_status_date_calendar_year_month, + 'placeholder text' AS project_development_status_date_calendar_year_month_numeric, + 'placeholder text' AS project_development_status_date_calendar_year_quarter, + 999 AS project_development_status_date_fiscal_year, + 'placeholder text' AS project_development_status_date_fiscal_year_quarter, + ( + SELECT project_name + FROM moped_project + WHERE project_id = mp.parent_project_id + ) AS parent_project_name, cpl.children_project_ids, string_agg(DISTINCT me2.entity_name, ', '::text) AS project_partners, - (SELECT JSON_AGG(json_build_object('signal_id', feature_signals.signal_id, 'knack_id', feature_signals.knack_id, 'location_name', feature_signals.location_name, 'signal_type', feature_signals.signal_type, 'id', feature_signals.id)) - FROM moped_proj_components components + ( + SELECT + json_agg( + json_build_object( + 'signal_id', + feature_signals.signal_id, + 'knack_id', + feature_signals.knack_id, + 'location_name', + feature_signals.location_name, + 'signal_type', + feature_signals.signal_type, + 'id', + feature_signals.id + ) + ) + FROM moped_proj_components AS components LEFT JOIN feature_signals - ON (feature_signals.component_id = components.project_component_id) + ON (components.project_component_id = feature_signals.component_id) WHERE TRUE - AND components.is_deleted = false - AND components.project_id = mp.project_id - AND feature_signals.signal_id is not null - AND feature_signals.is_deleted = false - ) as project_feature, + AND components.is_deleted = FALSE + AND components.project_id = mp.project_id + AND feature_signals.signal_id IS NOT NULL + AND feature_signals.is_deleted = FALSE + ) AS project_feature, fsl.funding_source_name, ptl.type_name, ( -- get the date of the construction phase with the earliest start date - SELECT min(phases.phase_start) - FROM moped_proj_phases phases - WHERE true - AND phases.project_id = mp.project_id - AND phases.phase_id = 9 -- phase_id 9 is construction - AND phases.is_deleted = false + SELECT min(phases.phase_start) + FROM moped_proj_phases AS phases + WHERE TRUE + AND phases.project_id = mp.project_id + AND phases.phase_id = 9 -- phase_id 9 is construction + AND phases.is_deleted = FALSE ) AS construction_start_date, ( -- get the date of the completion phase with the latest end date - SELECT max(phases.phase_end) - FROM moped_proj_phases phases - WHERE true - AND phases.project_id = mp.project_id - AND phases.phase_id = 11 -- phase_id 11 is complete - AND phases.is_deleted = false - ) AS completion_end_date, + SELECT max(phases.phase_end) + FROM moped_proj_phases AS phases + WHERE TRUE + AND phases.project_id = mp.project_id + AND phases.phase_id = 11 -- phase_id 11 is complete + AND phases.is_deleted = FALSE + ) AS completion_end_date, ( -- get the earliest confirmed phase_start or phase_end with a simple phase of 'Complete' - SELECT - min(min_confirmed_date) - FROM ( - -- earliest confirmed phase start - SELECT - min(phases.phase_start) AS min_confirmed_date - FROM - moped_proj_phases phases - LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id - WHERE - TRUE - AND phases.phase_start IS NOT NULL - AND phases.is_phase_start_confirmed = TRUE - AND phases.project_id = mp.project_id - AND moped_phases.phase_name_simple = 'Complete' - AND phases.is_deleted = FALSE - UNION ALL - -- earliest confirmed phase end - SELECT - min(phases.phase_end) AS min_confirmed_date - FROM - moped_proj_phases phases - LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id - WHERE - TRUE - AND phases.phase_end IS NOT NULL - AND phases.is_phase_end_confirmed = TRUE - AND phases.project_id = mp.project_id - AND moped_phases.phase_name_simple = 'Complete' - AND phases.is_deleted = FALSE - ) min_confirmed_dates - ) AS substantial_completion_date, + SELECT min(min_confirmed_date) + FROM ( + -- earliest confirmed phase start + SELECT min(phases.phase_start) AS min_confirmed_date + FROM + moped_proj_phases AS phases + LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id + WHERE + TRUE + AND phases.phase_start IS NOT NULL + AND phases.is_phase_start_confirmed = TRUE + AND phases.project_id = mp.project_id + AND moped_phases.phase_name_simple = 'Complete' + AND phases.is_deleted = FALSE + UNION ALL + -- earliest confirmed phase end + SELECT min(phases.phase_end) AS min_confirmed_date + FROM + moped_proj_phases AS phases + LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id + WHERE + TRUE + AND phases.phase_end IS NOT NULL + AND phases.is_phase_end_confirmed = TRUE + AND phases.project_id = mp.project_id + AND moped_phases.phase_name_simple = 'Complete' + AND phases.is_deleted = FALSE + ) AS min_confirmed_dates + ) AS substantial_completion_date, ( -- get me a list of the inspectors for this project - SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg - FROM moped_proj_personnel mpp - JOIN moped_users users ON mpp.user_id = users.user_id - JOIN moped_proj_personnel_roles mppr ON mpp.project_personnel_id = mppr.project_personnel_id - JOIN moped_project_roles mpr ON mppr.project_role_id = mpr.project_role_id - WHERE 1 = 1 - AND mpr.project_role_name = 'Inspector'::text - AND mpp.is_deleted = false - AND mppr.is_deleted = false - AND mpp.project_id = mp.project_id - GROUP BY mpp.project_id) AS project_inspector, + SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS users ON mpp.user_id = users.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE 1 = 1 + AND mpr.project_role_name = 'Inspector'::text + AND mpp.is_deleted = FALSE + AND mppr.is_deleted = FALSE + AND mpp.project_id = mp.project_id + GROUP BY mpp.project_id + ) AS project_inspector, ( -- get me a list of the designers for this project - SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg - FROM moped_proj_personnel mpp - JOIN moped_users users ON mpp.user_id = users.user_id - JOIN moped_proj_personnel_roles mppr ON mpp.project_personnel_id = mppr.project_personnel_id - JOIN moped_project_roles mpr ON mppr.project_role_id = mpr.project_role_id - WHERE 1 = 1 - AND mpr.project_role_name = 'Designer'::text - AND mpp.is_deleted = false - AND mppr.is_deleted = false - AND mpp.project_id = mp.project_id - GROUP BY mpp.project_id) AS project_designer, + SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS users ON mpp.user_id = users.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE 1 = 1 + AND mpr.project_role_name = 'Designer'::text + AND mpp.is_deleted = FALSE + AND mppr.is_deleted = FALSE + AND mpp.project_id = mp.project_id + GROUP BY mpp.project_id + ) AS project_designer, ( -- get me all of the tags added to a project - SELECT string_agg(tags.name, ', '::text) AS string_agg - FROM moped_proj_tags ptags - JOIN moped_tags tags ON ptags.tag_id = tags.id - WHERE 1 = 1 - AND ptags.is_deleted = false - AND ptags.project_id = mp.project_id - GROUP BY ptags.project_id) AS project_tags, + SELECT string_agg(tags.name, ', '::text) AS string_agg + FROM moped_proj_tags AS ptags + INNER JOIN moped_tags AS tags ON ptags.tag_id = tags.id + WHERE 1 = 1 + AND ptags.is_deleted = FALSE + AND ptags.project_id = mp.project_id + GROUP BY ptags.project_id + ) AS project_tags, concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, mpcs.components - FROM moped_project mp - LEFT JOIN project_person_list_lookup ppll ON mp.project_id = ppll.project_id - LEFT JOIN funding_sources_lookup fsl ON fsl.project_id = mp.project_id - LEFT JOIN project_type_lookup ptl ON ptl.project_id = mp.project_id - LEFT JOIN moped_entity me ON me.entity_id = mp.project_sponsor - LEFT JOIN moped_entity mel ON mel.entity_id = mp.project_lead_id - LEFT JOIN moped_proj_partners mpp2 ON mp.project_id = mpp2.project_id AND mpp2.is_deleted = false - LEFT JOIN moped_entity me2 ON mpp2.entity_id = me2.entity_id - LEFT JOIN work_activities on work_activities.project_id = mp.project_id - LEFT JOIN moped_users added_by_user ON mp.added_by = added_by_user.user_id - LEFT JOIN current_phase_view current_phase on mp.project_id = current_phase.project_id - LEFT JOIN moped_public_process_statuses mpps ON mpps.id = mp.public_process_status_id - LEFT JOIN child_project_lookup cpl on cpl.parent_id = mp.project_id - LEFT JOIN moped_proj_components_subtypes mpcs on mpcs.project_id = mp.project_id - LEFT JOIN LATERAL - ( - SELECT mpn.project_note, mpn.date_created - FROM moped_proj_notes mpn - WHERE mpn.project_id = mp.project_id AND mpn.project_note_type = 2 AND mpn.is_deleted = false +FROM moped_project AS mp +LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id +LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id +LEFT JOIN project_type_lookup AS ptl ON mp.project_id = ptl.project_id +LEFT JOIN moped_entity AS me ON mp.project_sponsor = me.entity_id +LEFT JOIN moped_entity AS mel ON mp.project_lead_id = mel.entity_id +LEFT JOIN moped_proj_partners AS mpp2 ON mp.project_id = mpp2.project_id AND mpp2.is_deleted = FALSE +LEFT JOIN moped_entity AS me2 ON mpp2.entity_id = me2.entity_id +LEFT JOIN work_activities ON mp.project_id = work_activities.project_id +LEFT JOIN moped_users AS added_by_user ON mp.added_by = added_by_user.user_id +LEFT JOIN current_phase_view AS current_phase ON mp.project_id = current_phase.project_id +LEFT JOIN moped_public_process_statuses AS mpps ON mp.public_process_status_id = mpps.id +LEFT JOIN child_project_lookup AS cpl ON mp.project_id = cpl.parent_id +LEFT JOIN moped_proj_components_subtypes AS mpcs ON mp.project_id = mpcs.project_id +LEFT JOIN LATERAL + ( + SELECT + mpn.project_note, + mpn.date_created + FROM moped_proj_notes AS mpn + WHERE mpn.project_id = mp.project_id AND mpn.project_note_type = 2 AND mpn.is_deleted = FALSE ORDER BY mpn.date_created DESC LIMIT 1 - ) as proj_status_update on true - WHERE - mp.is_deleted = false - GROUP BY - mp.project_id, - mp.project_name, - mp.project_description, - ppll.project_team_members, - mp.ecapris_subproject_id, + ) AS proj_status_update ON TRUE +WHERE + mp.is_deleted = FALSE +GROUP BY + mp.project_id, + mp.project_name, + mp.project_description, + ppll.project_team_members, + mp.ecapris_subproject_id, mp.date_added, mp.is_deleted, me.entity_name, @@ -487,77 +534,216 @@ AS WITH project_person_list_lookup AS ( work_activities.task_orders; CREATE OR REPLACE VIEW component_arcgis_online_view AS ( - SELECT - mpc.project_id, - comp_geography.project_component_id, - comp_geography.feature_ids, - mpc.component_id, - comp_geography.geometry, + WITH work_types AS ( + -- group work types by project component id + SELECT + project_component_id, + string_agg(mwt.name, ', ') AS work_types + FROM + moped_proj_component_work_types AS mpcwt + LEFT JOIN moped_work_types AS mwt ON mpcwt.work_type_id = mwt.id + WHERE mpcwt.is_deleted = FALSE + GROUP BY + project_component_id + ), +council_districts AS ( + -- group council districts by project component id + SELECT + component_id AS project_component_id, + string_agg( + DISTINCT council_district_id::text, + ', ' + ) AS council_districts + FROM + features_council_districts + LEFT JOIN features ON features_council_districts.feature_id = features.id + WHERE + features.is_deleted = FALSE + GROUP BY + component_id + ), +comp_geography AS ( + -- group feature properties by project component ID + SELECT + component_id AS project_component_id, + string_agg(DISTINCT id::text, ', ') AS feature_ids, + st_asgeojson( + st_union( + array_agg(geography) + ) + )::json AS "geometry", + st_asgeojson( + st_union( + array_agg(line_geography) + ) + )::json AS "line_geometry", + string_agg(DISTINCT signal_id::text, ', ') AS signal_ids, + sum(length_feet) AS length_feet_total + FROM + ( + -- union all features + SELECT + id, + feature_signals.component_id, + feature_signals.geography::geometry, + st_exteriorring( + st_buffer(feature_signals.geography, 7)::geometry + ) AS line_geography, + feature_signals.signal_id, + NULL AS length_feet + FROM + feature_signals + WHERE + feature_signals.is_deleted = FALSE + UNION ALL + SELECT + id, + feature_street_segments.component_id, + feature_street_segments.geography::geometry, + feature_street_segments.geography::geometry AS line_geography, + NULL AS signal_id, + length_feet + FROM + feature_street_segments + WHERE + feature_street_segments.is_deleted = FALSE + UNION ALL + SELECT + id, + feature_intersections.component_id, + feature_intersections.geography::geometry, + st_exteriorring( + st_buffer(feature_intersections.geography, 7)::geometry + ) AS line_geography, + NULL AS signal_id, + NULL AS length_feet + FROM + feature_intersections + WHERE + feature_intersections.is_deleted = FALSE + UNION ALL + SELECT + id, + feature_drawn_points.component_id, + feature_drawn_points.geography::geometry, + st_exteriorring( + st_buffer(feature_drawn_points.geography, 7)::geometry + ) AS line_geography, + NULL AS signal_id, + NULL AS length_feet + FROM + feature_drawn_points + WHERE + feature_drawn_points.is_deleted = FALSE + UNION ALL + SELECT + id, + feature_drawn_lines.component_id, + feature_drawn_lines.geography::geometry, + feature_drawn_lines.geography::geometry AS line_geography, + NULL AS signal_id, + length_feet + FROM + feature_drawn_lines + WHERE + feature_drawn_lines.is_deleted = FALSE + ) AS feature_union + GROUP BY + component_id + ), +subcomponents AS ( + -- group subcomponents by project component id + SELECT + project_component_id, + string_agg(ms.subcomponent_name, ', ') AS subcomponents + FROM + moped_proj_components_subcomponents AS mpcs + LEFT JOIN moped_subcomponents AS ms ON mpcs.subcomponent_id = ms.subcomponent_id + WHERE mpcs.is_deleted = FALSE + GROUP BY + project_component_id + ), +component_tags AS ( + -- group project component tags by project component id + SELECT + project_component_id, + string_agg(mct.type || ' - ' || mct.name, ', ') AS component_tags + FROM + moped_proj_component_tags AS mpct + LEFT JOIN moped_component_tags AS mct ON mpct.component_tag_id = mct.id + WHERE mpct.is_deleted = FALSE + GROUP BY + project_component_id + ) + SELECT + mpc.project_id, + comp_geography.project_component_id, + comp_geography.feature_ids, + mpc.component_id, + comp_geography.geometry, comp_geography.line_geometry, - comp_geography.signal_ids, - council_districts.council_districts, - 'placeholder text' as council_districts_searchable, - CASE WHEN council_districts IS NULL OR council_districts = '' THEN FALSE - ELSE TRUE - END AS is_within_city_limits, - comp_geography.length_feet_total, - round((comp_geography.length_feet_total / 5280::numeric),2) AS length_miles_total, - mc.component_name, - mc.component_subtype, - mc.component_name_full, - 'placeholder text' as component_categories, - subcomponents.subcomponents as component_subcomponents, - work_types.work_types as component_work_types, - component_tags.component_tags, + comp_geography.signal_ids, + council_districts.council_districts, + 'placeholder text' AS council_districts_searchable, + NOT coalesce (council_districts IS NULL OR council_districts = '', FALSE) AS is_within_city_limits, + comp_geography.length_feet_total, + round((comp_geography.length_feet_total / 5280::numeric), 2) AS length_miles_total, + mc.component_name, + mc.component_subtype, + mc.component_name_full, + 'placeholder text' AS component_categories, + subcomponents.subcomponents AS component_subcomponents, + work_types.work_types AS component_work_types, + component_tags.component_tags, mpc.description AS component_description, - mpc.interim_project_component_id, + mpc.interim_project_component_id, mpc.completion_date, - COALESCE(mpc.completion_date, substantial_completion_date) as substantial_completion_date, - '2024-01-01T00:00:00-06:00' as substantial_completion_date_estimated, - mpc.srts_id, - mpc.location_description as component_location_description, + coalesce(mpc.completion_date, substantial_completion_date) AS substantial_completion_date, + '2024-01-01T00:00:00-06:00' AS substantial_completion_date_estimated, + mpc.srts_id, + mpc.location_description AS component_location_description, plv.project_name, - 'placeholder text' as project_name_descriptor, - 'placeholder text' as project_name_with_descriptor, - plv.project_description, - plv.ecapris_subproject_id, + 'placeholder text' AS project_name_descriptor, + 'placeholder text' AS project_name_with_descriptor, + plv.project_description, + plv.ecapris_subproject_id, plv.project_website, - plv.updated_at as project_updated_at, - mpc.phase_id AS component_phase_id, - mph.phase_name AS component_phase_name, - mph.phase_name_simple as component_phase_name_simple, - current_phase.phase_id AS project_phase_id, - current_phase.phase_name AS project_phase_name, - current_phase.phase_name_simple AS project_phase_name_simple, - COALESCE(mph.phase_name, current_phase.phase_name) AS current_phase_name, - COALESCE(mph.phase_name_simple, current_phase.phase_name_simple) AS current_phase_name_simple, - plv.project_team_members, - plv.project_sponsor, - plv.project_lead, - plv.public_process_status, - plv.interim_project_id, - plv.project_partners, - plv.task_order_names, - plv.funding_source_name, - plv.type_name, - plv.project_status_update, - plv.project_status_update_date_created, - plv.construction_start_date, - plv.completion_end_date, - plv.project_inspector, - plv.project_designer, - plv.project_tags, - plv.workgroup_contractors, - plv.contract_numbers, + plv.updated_at AS project_updated_at, + mpc.phase_id AS component_phase_id, + mph.phase_name AS component_phase_name, + mph.phase_name_simple AS component_phase_name_simple, + current_phase.phase_id AS project_phase_id, + current_phase.phase_name AS project_phase_name, + current_phase.phase_name_simple AS project_phase_name_simple, + coalesce(mph.phase_name, current_phase.phase_name) AS current_phase_name, + coalesce(mph.phase_name_simple, current_phase.phase_name_simple) AS current_phase_name_simple, + plv.project_team_members, + plv.project_sponsor, + plv.project_lead, + plv.public_process_status, + plv.interim_project_id, + plv.project_partners, + plv.task_order_names, + plv.funding_source_name, + plv.type_name, + plv.project_status_update, + plv.project_status_update_date_created, + plv.construction_start_date, + plv.completion_end_date, + plv.project_inspector, + plv.project_designer, + plv.project_tags, + plv.workgroup_contractors, + plv.contract_numbers, plv.parent_project_id, plv.parent_project_name, plv.parent_project_url, - 'placeholder text' as parent_project_name_with_descriptor, - 'placeholder text' as related_project_ids, - 'placeholder text' as related_project_ids_searchable, - plv.knack_project_id as knack_data_tracker_project_record_id, + 'placeholder text' AS parent_project_name_with_descriptor, + 'placeholder text' AS related_project_ids, + 'placeholder text' AS related_project_ids_searchable, + plv.knack_project_id AS knack_data_tracker_project_record_id, plv.project_url, - plv.project_url || '?tab=map&project_component_id=' || mpc.project_component_id :: text as component_url, + plv.project_url || '?tab=map&project_component_id=' || mpc.project_component_id::text AS component_url, plv.project_development_status, plv.project_development_status_date, plv.project_development_status_date_calendar_year, @@ -566,155 +752,19 @@ CREATE OR REPLACE VIEW component_arcgis_online_view AS ( plv.project_development_status_date_calendar_year_quarter, plv.project_development_status_date_fiscal_year, plv.project_development_status_date_fiscal_year_quarter, - added_by as project_added_by - FROM - moped_proj_components mpc - LEFT JOIN ( - -- group feature properties by project component ID - SELECT - component_id AS project_component_id, - STRING_AGG(DISTINCT id :: text, ', ') AS feature_ids, - ST_AsGeoJSON( - ST_Union( - ARRAY_AGG(geography) - ) - ):: json AS "geometry", - ST_AsGeoJSON( - ST_Union( - ARRAY_AGG(line_geography) - ) - ):: json AS "line_geometry", - STRING_AGG(DISTINCT signal_id :: text, ', ') AS signal_ids, - SUM(length_feet) as length_feet_total - FROM - ( - -- union all features - SELECT - id, - feature_signals.component_id, - feature_signals.geography :: geometry, - ST_ExteriorRing( - ST_Buffer(feature_signals.geography, 7):: geometry - ) AS line_geography, - feature_signals.signal_id, - NULL AS length_feet - FROM - feature_signals - WHERE - feature_signals.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_street_segments.component_id, - feature_street_segments.geography :: geometry, - feature_street_segments.geography :: geometry as line_geography, - NULL AS signal_id, - length_feet - FROM - feature_street_segments - WHERE - feature_street_segments.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_intersections.component_id, - feature_intersections.geography :: geometry, - ST_ExteriorRing( - ST_Buffer(feature_intersections.geography, 7):: geometry - ) AS line_geography, - NULL AS signal_id, - NULL AS length_feet - FROM - feature_intersections - WHERE - feature_intersections.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_drawn_points.component_id, - feature_drawn_points.geography :: geometry, - ST_ExteriorRing( - ST_Buffer(feature_drawn_points.geography, 7):: geometry - ) AS line_geography, - NULL AS signal_id, - NULL AS length_feet - FROM - feature_drawn_points - WHERE - feature_drawn_points.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_drawn_lines.component_id, - feature_drawn_lines.geography :: geometry, - feature_drawn_lines.geography :: geometry as line_geography, - NULL AS signal_id, - length_feet - FROM - feature_drawn_lines - WHERE - feature_drawn_lines.is_deleted = FALSE - ) feature_union - GROUP BY - component_id - ) comp_geography ON comp_geography.project_component_id = mpc.project_component_id - LEFT JOIN ( - -- group council districts by project component id - SELECT - component_id AS project_component_id, - STRING_AGG( - DISTINCT council_district_id :: text, - ', ' - ) AS council_districts - FROM - features_council_districts - LEFT JOIN features ON features.id = features_council_districts.feature_id - WHERE - features.is_deleted = FALSE - GROUP BY - component_id - ) council_districts ON council_districts.project_component_id = mpc.project_component_id - LEFT JOIN ( - -- group subcomponents by project component id - SELECT - project_component_id, - string_agg(ms.subcomponent_name, ', ') subcomponents - FROM - moped_proj_components_subcomponents mpcs - LEFT JOIN moped_subcomponents ms ON mpcs.subcomponent_id = ms.subcomponent_id - WHERE mpcs.is_deleted = FALSE - GROUP BY - project_component_id - ) subcomponents ON subcomponents.project_component_id = mpc.project_component_id - LEFT JOIN ( - -- group work types by project component id - SELECT - project_component_id, - string_agg(mwt.name, ', ') work_types - FROM - moped_proj_component_work_types mpcwt - LEFT JOIN moped_work_types mwt ON mpcwt.work_type_id = mwt.id - WHERE mpcwt.is_deleted = FALSE - GROUP BY - project_component_id - ) work_types ON work_types.project_component_id = mpc.project_component_id - LEFT JOIN ( - -- group project component tags by project component id - SELECT - project_component_id, - string_agg(mct.type || ' - ' || mct.name, ', ') component_tags - FROM - moped_proj_component_tags mpct - LEFT JOIN moped_component_tags mct ON mpct.component_tag_id = mct.id - WHERE mpct.is_deleted = FALSE - GROUP BY - project_component_id - ) component_tags ON component_tags.project_component_id = mpc.project_component_id - LEFT JOIN project_list_view plv ON plv.project_id = mpc.project_id - LEFT JOIN current_phase_view current_phase ON mpc.project_id = current_phase.project_id - LEFT JOIN moped_phases mph ON mpc.phase_id = mph.phase_id - LEFT JOIN moped_components mc ON mc.component_id = mpc.component_id -WHERE - mpc.is_deleted = FALSE - AND plv.is_deleted = FALSE + added_by AS project_added_by + FROM + moped_proj_components AS mpc + LEFT JOIN comp_geography ON mpc.project_component_id = comp_geography.project_component_id + LEFT JOIN council_districts ON mpc.project_component_id = council_districts.project_component_id + LEFT JOIN subcomponents ON mpc.project_component_id = subcomponents.project_component_id + LEFT JOIN work_types ON mpc.project_component_id = work_types.project_component_id + LEFT JOIN component_tags ON mpc.project_component_id = component_tags.project_component_id + LEFT JOIN project_list_view AS plv ON mpc.project_id = plv.project_id + LEFT JOIN current_phase_view AS current_phase ON mpc.project_id = current_phase.project_id + LEFT JOIN moped_phases AS mph ON mpc.phase_id = mph.phase_id + LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id + WHERE + mpc.is_deleted = FALSE + AND plv.is_deleted = FALSE ); diff --git a/moped-database/views/current_phase_view.sql b/moped-database/views/current_phase_view.sql index 482115ede4..110d16b425 100644 --- a/moped-database/views/current_phase_view.sql +++ b/moped-database/views/current_phase_view.sql @@ -7,14 +7,14 @@ CREATE VIEW public.current_phase_view AS ( mp.phase_key, mp.phase_name_simple FROM - moped_proj_phases mpp - LEFT JOIN moped_phases mp ON mp.phase_id = mpp.phase_id -WHERE - is_deleted = FALSE - AND is_current_phase = TRUE + moped_proj_phases AS mpp + LEFT JOIN moped_phases AS mp ON mpp.phase_id = mp.phase_id + WHERE + is_deleted = FALSE + AND is_current_phase = TRUE -- order by ensures consistent DISTINCT ON results in edge case that a project has multiple curr phases -ORDER BY - project_id, - is_current_phase, - project_phase_id + ORDER BY + project_id, + is_current_phase, + project_phase_id ); diff --git a/moped-database/views/project_geography.sql b/moped-database/views/project_geography.sql index a38f8dacff..7561f3c505 100644 --- a/moped-database/views/project_geography.sql +++ b/moped-database/views/project_geography.sql @@ -1,7 +1,8 @@ -- latest version 1700149643859_add_audit_fields_to_unified_features_view CREATE OR REPLACE VIEW public.project_geography -AS SELECT moped_project.project_id, +AS SELECT + moped_project.project_id, uniform_features.id AS feature_id, moped_components.component_id AS component_archtype_id, moped_proj_components.project_component_id AS component_id, @@ -14,14 +15,14 @@ AS SELECT moped_project.project_id, uniform_features.geography, uniform_features.council_districts, uniform_features.length_feet, - uniform_features.created_at as feature_created_at, - uniform_features.updated_at as feature_updated_at, - uniform_features.created_by_user_id as feature_created_by_user_id, - uniform_features.updated_by_user_id as feature_updated_by_user_id - FROM moped_project - JOIN moped_proj_components ON moped_proj_components.project_id = moped_project.project_id - JOIN moped_components ON moped_proj_components.component_id = moped_components.component_id - JOIN feature_layers ON moped_components.feature_layer_id = feature_layers.id - JOIN uniform_features ON moped_proj_components.project_component_id = uniform_features.component_id; - + uniform_features.created_at AS feature_created_at, + uniform_features.updated_at AS feature_updated_at, + uniform_features.created_by_user_id AS feature_created_by_user_id, + uniform_features.updated_by_user_id AS feature_updated_by_user_id +FROM moped_project +INNER JOIN moped_proj_components ON moped_project.project_id = moped_proj_components.project_id +INNER JOIN moped_components ON moped_proj_components.component_id = moped_components.component_id +INNER JOIN feature_layers ON moped_components.feature_layer_id = feature_layers.id +INNER JOIN uniform_features ON moped_proj_components.project_component_id = uniform_features.component_id; + COMMENT ON VIEW public.project_geography IS 'The project_geography view merges project-specific data with the unified geographical features from the uniform_features view. It links projects with their respective geographical components, including type, attributes, and location.'; diff --git a/moped-database/views/project_list_view.sql b/moped-database/views/project_list_view.sql index d34ff47001..1e73691c4a 100644 --- a/moped-database/views/project_list_view.sql +++ b/moped-database/views/project_list_view.sql @@ -2,63 +2,89 @@ CREATE OR REPLACE VIEW public.project_list_view AS WITH project_person_list_lookup AS ( SELECT - mpp.project_id, - string_agg(DISTINCT concat(mu.first_name, ' ', mu.last_name, ':', mpr.project_role_name), ','::text) AS project_team_members - FROM moped_proj_personnel mpp - JOIN moped_users mu ON mpp.user_id = mu.user_id - JOIN moped_proj_personnel_roles mppr ON mpp.project_personnel_id = mppr.project_personnel_id - JOIN moped_project_roles mpr ON mppr.project_role_id = mpr.project_role_id + mpp.project_id, + string_agg( + DISTINCT concat(mu.first_name, ' ', mu.last_name, ':', mpr.project_role_name), ','::text + ) AS project_team_members + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS mu ON mpp.user_id = mu.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id WHERE mpp.is_deleted = false - AND mppr.is_deleted = false + AND mppr.is_deleted = false GROUP BY mpp.project_id - ), funding_sources_lookup AS ( - SELECT - mpf_1.project_id, - string_agg(mfs.funding_source_name, ', '::text) AS funding_source_name - FROM moped_proj_funding mpf_1 - LEFT JOIN moped_fund_sources mfs ON mpf_1.funding_source_id = mfs.funding_source_id +), + +funding_sources_lookup AS ( + SELECT + mpf_1.project_id, + string_agg(mfs.funding_source_name, ', '::text) AS funding_source_name + FROM moped_proj_funding AS mpf_1 + LEFT JOIN moped_fund_sources AS mfs ON mpf_1.funding_source_id = mfs.funding_source_id WHERE mpf_1.is_deleted = false GROUP BY mpf_1.project_id - ), project_type_lookup AS ( +), + +project_type_lookup AS ( SELECT - mpt.project_id, - string_agg(mt.type_name, ', '::text) AS type_name - FROM moped_project_types mpt - LEFT JOIN moped_types mt ON mpt.project_type_id = mt.type_id AND mpt.is_deleted = false + mpt.project_id, + string_agg(mt.type_name, ', '::text) AS type_name + FROM moped_project_types AS mpt + LEFT JOIN moped_types AS mt ON mpt.project_type_id = mt.type_id AND mpt.is_deleted = false GROUP BY mpt.project_id - ), child_project_lookup AS ( - SELECT jsonb_agg(children.project_id) AS children_project_ids, - children.parent_project_id AS parent_id - FROM moped_project AS children - JOIN moped_project AS parent ON (parent.project_id = children.parent_project_id) - WHERE children.is_deleted = false +), + +child_project_lookup AS ( + SELECT + jsonb_agg(children.project_id) AS children_project_ids, + children.parent_project_id AS parent_id + FROM moped_project AS children + INNER JOIN moped_project AS parent ON (children.parent_project_id = parent.project_id) + WHERE children.is_deleted = false GROUP BY parent_id - ), work_activities AS ( - SELECT - project_id, - string_agg(task_order_objects.task_order_object ->> 'display_name'::text, - ', '::text) AS task_order_names, - string_agg(task_order_objects.task_order_object ->> 'task_order'::text, - ', '::text) AS task_order_names_short, - jsonb_agg(DISTINCT task_order_objects.task_order_object) FILTER (WHERE task_order_objects.task_order_object IS NOT NULL) AS task_orders, - string_agg(DISTINCT mpwa.workgroup_contractor, - ', '::text) AS workgroup_contractors, - string_agg(mpwa.contract_number, - ', '::text) AS contract_numbers FROM moped_proj_work_activity mpwa - LEFT JOIN LATERAL jsonb_array_elements(mpwa.task_orders) task_order_objects (task_order_object) ON TRUE WHERE 1 = 1 - AND mpwa.is_deleted = FALSE - GROUP BY - mpwa.project_id - ), moped_proj_components_subtypes AS ( +), + +work_activities AS ( SELECT - mpc.project_id, - string_agg(DISTINCT mc.component_name_full, ', '::text) AS components - FROM moped_proj_components mpc - LEFT JOIN moped_components mc ON mpc.component_id = mc.component_id - WHERE mpc.is_deleted = FALSE + project_id, + string_agg( + task_order_objects.task_order_object ->> 'display_name'::text, + ', '::text + ) AS task_order_names, + string_agg( + task_order_objects.task_order_object ->> 'task_order'::text, + ', '::text + ) AS task_order_names_short, + jsonb_agg(DISTINCT task_order_objects.task_order_object) FILTER ( + WHERE task_order_objects.task_order_object IS NOT null + ) AS task_orders, + string_agg( + DISTINCT mpwa.workgroup_contractor, + ', '::text + ) AS workgroup_contractors, + string_agg( + mpwa.contract_number, + ', '::text + ) AS contract_numbers + FROM moped_proj_work_activity AS mpwa + LEFT JOIN LATERAL jsonb_array_elements(mpwa.task_orders) AS task_order_objects (task_order_object) + ON true WHERE 1 = 1 + AND mpwa.is_deleted = false + GROUP BY + mpwa.project_id +), + +moped_proj_components_subtypes AS ( + SELECT + mpc.project_id, + string_agg(DISTINCT mc.component_name_full, ', '::text) AS components + FROM moped_proj_components AS mpc + LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id + WHERE mpc.is_deleted = false GROUP BY mpc.project_id - ) - SELECT +) + +SELECT mp.project_id, mp.project_name, mp.project_description, @@ -67,9 +93,9 @@ AS WITH project_person_list_lookup AS ( mp.date_added, mp.is_deleted, mp.updated_at, - current_phase.phase_name as current_phase, - current_phase.phase_key as current_phase_key, - current_phase.phase_name_simple as current_phase_simple, + current_phase.phase_name AS current_phase, + current_phase.phase_key AS current_phase_key, + current_phase.phase_name_simple AS current_phase_simple, ppll.project_team_members, me.entity_name AS project_sponsor, mel.entity_name AS project_lead, @@ -77,154 +103,172 @@ AS WITH project_person_list_lookup AS ( mp.interim_project_id, mp.parent_project_id, mp.knack_project_id, - 'https://mobility.austin.gov/moped/projects/' || mp.project_id :: text as project_url, - 'https://mobility.austin.gov/moped/projects/' || mp.parent_project_id :: text as parent_project_url, - proj_status_update.project_note as project_status_update, - proj_status_update.date_created as project_status_update_date_created, + 'https://mobility.austin.gov/moped/projects/' || mp.project_id::text AS project_url, + 'https://mobility.austin.gov/moped/projects/' || mp.parent_project_id::text AS parent_project_url, + proj_status_update.project_note AS project_status_update, + proj_status_update.date_created AS project_status_update_date_created, work_activities.workgroup_contractors, work_activities.contract_numbers, work_activities.task_order_names, work_activities.task_order_names_short, work_activities.task_orders, - 'placeholder text' as project_development_status, - '2024-01-01T00:00:00-06:00' as project_development_status_date, - 9999 as project_development_status_date_calendar_year, - 'placeholder text' as project_development_status_date_calendar_year_month, - 'placeholder text' as project_development_status_date_calendar_year_month_numeric, - 'placeholder text' as project_development_status_date_calendar_year_quarter, - 999 as project_development_status_date_fiscal_year, - 'placeholder text' as project_development_status_date_fiscal_year_quarter, - (SELECT project_name - FROM moped_project - WHERE project_id = mp.parent_project_id - ) as parent_project_name, + 'placeholder text' AS project_development_status, + '2024-01-01T00:00:00-06:00' AS project_development_status_date, + 9999 AS project_development_status_date_calendar_year, + 'placeholder text' AS project_development_status_date_calendar_year_month, + 'placeholder text' AS project_development_status_date_calendar_year_month_numeric, + 'placeholder text' AS project_development_status_date_calendar_year_quarter, + 999 AS project_development_status_date_fiscal_year, + 'placeholder text' AS project_development_status_date_fiscal_year_quarter, + ( + SELECT project_name + FROM moped_project + WHERE project_id = mp.parent_project_id + ) AS parent_project_name, cpl.children_project_ids, string_agg(DISTINCT me2.entity_name, ', '::text) AS project_partners, - (SELECT JSON_AGG(json_build_object('signal_id', feature_signals.signal_id, 'knack_id', feature_signals.knack_id, 'location_name', feature_signals.location_name, 'signal_type', feature_signals.signal_type, 'id', feature_signals.id)) - FROM moped_proj_components components + ( + SELECT + json_agg( + json_build_object( + 'signal_id', + feature_signals.signal_id, + 'knack_id', + feature_signals.knack_id, + 'location_name', + feature_signals.location_name, + 'signal_type', + feature_signals.signal_type, + 'id', + feature_signals.id + ) + ) + FROM moped_proj_components AS components LEFT JOIN feature_signals - ON (feature_signals.component_id = components.project_component_id) - WHERE TRUE - AND components.is_deleted = false - AND components.project_id = mp.project_id - AND feature_signals.signal_id is not null - AND feature_signals.is_deleted = false - ) as project_feature, + ON (components.project_component_id = feature_signals.component_id) + WHERE true + AND components.is_deleted = false + AND components.project_id = mp.project_id + AND feature_signals.signal_id IS NOT null + AND feature_signals.is_deleted = false + ) AS project_feature, fsl.funding_source_name, ptl.type_name, ( -- get the date of the construction phase with the earliest start date - SELECT min(phases.phase_start) - FROM moped_proj_phases phases - WHERE true - AND phases.project_id = mp.project_id - AND phases.phase_id = 9 -- phase_id 9 is construction - AND phases.is_deleted = false + SELECT min(phases.phase_start) + FROM moped_proj_phases AS phases + WHERE true + AND phases.project_id = mp.project_id + AND phases.phase_id = 9 -- phase_id 9 is construction + AND phases.is_deleted = false ) AS construction_start_date, ( -- get the date of the completion phase with the latest end date - SELECT max(phases.phase_end) - FROM moped_proj_phases phases - WHERE true - AND phases.project_id = mp.project_id - AND phases.phase_id = 11 -- phase_id 11 is complete - AND phases.is_deleted = false - ) AS completion_end_date, + SELECT max(phases.phase_end) + FROM moped_proj_phases AS phases + WHERE true + AND phases.project_id = mp.project_id + AND phases.phase_id = 11 -- phase_id 11 is complete + AND phases.is_deleted = false + ) AS completion_end_date, ( -- get the earliest confirmed phase_start or phase_end with a simple phase of 'Complete' - SELECT - min(min_confirmed_date) - FROM ( - -- earliest confirmed phase start - SELECT - min(phases.phase_start) AS min_confirmed_date - FROM - moped_proj_phases phases - LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id - WHERE - TRUE - AND phases.phase_start IS NOT NULL - AND phases.is_phase_start_confirmed = TRUE - AND phases.project_id = mp.project_id - AND moped_phases.phase_name_simple = 'Complete' - AND phases.is_deleted = FALSE - UNION ALL - -- earliest confirmed phase end - SELECT - min(phases.phase_end) AS min_confirmed_date - FROM - moped_proj_phases phases - LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id - WHERE - TRUE - AND phases.phase_end IS NOT NULL - AND phases.is_phase_end_confirmed = TRUE - AND phases.project_id = mp.project_id - AND moped_phases.phase_name_simple = 'Complete' - AND phases.is_deleted = FALSE - ) min_confirmed_dates - ) AS substantial_completion_date, + SELECT min(min_confirmed_date) + FROM ( + -- earliest confirmed phase start + SELECT min(phases.phase_start) AS min_confirmed_date + FROM + moped_proj_phases AS phases + LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id + WHERE + true + AND phases.phase_start IS NOT null + AND phases.is_phase_start_confirmed = true + AND phases.project_id = mp.project_id + AND moped_phases.phase_name_simple = 'Complete' + AND phases.is_deleted = false + UNION ALL + -- earliest confirmed phase end + SELECT min(phases.phase_end) AS min_confirmed_date + FROM + moped_proj_phases AS phases + LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id + WHERE + true + AND phases.phase_end IS NOT null + AND phases.is_phase_end_confirmed = true + AND phases.project_id = mp.project_id + AND moped_phases.phase_name_simple = 'Complete' + AND phases.is_deleted = false + ) AS min_confirmed_dates + ) AS substantial_completion_date, ( -- get me a list of the inspectors for this project - SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg - FROM moped_proj_personnel mpp - JOIN moped_users users ON mpp.user_id = users.user_id - JOIN moped_proj_personnel_roles mppr ON mpp.project_personnel_id = mppr.project_personnel_id - JOIN moped_project_roles mpr ON mppr.project_role_id = mpr.project_role_id - WHERE 1 = 1 - AND mpr.project_role_name = 'Inspector'::text - AND mpp.is_deleted = false - AND mppr.is_deleted = false - AND mpp.project_id = mp.project_id - GROUP BY mpp.project_id) AS project_inspector, + SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS users ON mpp.user_id = users.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE 1 = 1 + AND mpr.project_role_name = 'Inspector'::text + AND mpp.is_deleted = false + AND mppr.is_deleted = false + AND mpp.project_id = mp.project_id + GROUP BY mpp.project_id + ) AS project_inspector, ( -- get me a list of the designers for this project - SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg - FROM moped_proj_personnel mpp - JOIN moped_users users ON mpp.user_id = users.user_id - JOIN moped_proj_personnel_roles mppr ON mpp.project_personnel_id = mppr.project_personnel_id - JOIN moped_project_roles mpr ON mppr.project_role_id = mpr.project_role_id - WHERE 1 = 1 - AND mpr.project_role_name = 'Designer'::text - AND mpp.is_deleted = false - AND mppr.is_deleted = false - AND mpp.project_id = mp.project_id - GROUP BY mpp.project_id) AS project_designer, + SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS users ON mpp.user_id = users.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE 1 = 1 + AND mpr.project_role_name = 'Designer'::text + AND mpp.is_deleted = false + AND mppr.is_deleted = false + AND mpp.project_id = mp.project_id + GROUP BY mpp.project_id + ) AS project_designer, ( -- get me all of the tags added to a project - SELECT string_agg(tags.name, ', '::text) AS string_agg - FROM moped_proj_tags ptags - JOIN moped_tags tags ON ptags.tag_id = tags.id - WHERE 1 = 1 - AND ptags.is_deleted = false - AND ptags.project_id = mp.project_id - GROUP BY ptags.project_id) AS project_tags, + SELECT string_agg(tags.name, ', '::text) AS string_agg + FROM moped_proj_tags AS ptags + INNER JOIN moped_tags AS tags ON ptags.tag_id = tags.id + WHERE 1 = 1 + AND ptags.is_deleted = false + AND ptags.project_id = mp.project_id + GROUP BY ptags.project_id + ) AS project_tags, concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, mpcs.components - FROM moped_project mp - LEFT JOIN project_person_list_lookup ppll ON mp.project_id = ppll.project_id - LEFT JOIN funding_sources_lookup fsl ON fsl.project_id = mp.project_id - LEFT JOIN project_type_lookup ptl ON ptl.project_id = mp.project_id - LEFT JOIN moped_entity me ON me.entity_id = mp.project_sponsor - LEFT JOIN moped_entity mel ON mel.entity_id = mp.project_lead_id - LEFT JOIN moped_proj_partners mpp2 ON mp.project_id = mpp2.project_id AND mpp2.is_deleted = false - LEFT JOIN moped_entity me2 ON mpp2.entity_id = me2.entity_id - LEFT JOIN work_activities on work_activities.project_id = mp.project_id - LEFT JOIN moped_users added_by_user ON mp.added_by = added_by_user.user_id - LEFT JOIN current_phase_view current_phase on mp.project_id = current_phase.project_id - LEFT JOIN moped_public_process_statuses mpps ON mpps.id = mp.public_process_status_id - LEFT JOIN child_project_lookup cpl on cpl.parent_id = mp.project_id - LEFT JOIN moped_proj_components_subtypes mpcs on mpcs.project_id = mp.project_id - LEFT JOIN LATERAL - ( - SELECT mpn.project_note, mpn.date_created - FROM moped_proj_notes mpn +FROM moped_project AS mp +LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id +LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id +LEFT JOIN project_type_lookup AS ptl ON mp.project_id = ptl.project_id +LEFT JOIN moped_entity AS me ON mp.project_sponsor = me.entity_id +LEFT JOIN moped_entity AS mel ON mp.project_lead_id = mel.entity_id +LEFT JOIN moped_proj_partners AS mpp2 ON mp.project_id = mpp2.project_id AND mpp2.is_deleted = false +LEFT JOIN moped_entity AS me2 ON mpp2.entity_id = me2.entity_id +LEFT JOIN work_activities ON mp.project_id = work_activities.project_id +LEFT JOIN moped_users AS added_by_user ON mp.added_by = added_by_user.user_id +LEFT JOIN current_phase_view AS current_phase ON mp.project_id = current_phase.project_id +LEFT JOIN moped_public_process_statuses AS mpps ON mp.public_process_status_id = mpps.id +LEFT JOIN child_project_lookup AS cpl ON mp.project_id = cpl.parent_id +LEFT JOIN moped_proj_components_subtypes AS mpcs ON mp.project_id = mpcs.project_id +LEFT JOIN LATERAL + ( + SELECT + mpn.project_note, + mpn.date_created + FROM moped_proj_notes AS mpn WHERE mpn.project_id = mp.project_id AND mpn.project_note_type = 2 AND mpn.is_deleted = false ORDER BY mpn.date_created DESC LIMIT 1 - ) as proj_status_update on true - WHERE + ) AS proj_status_update ON true +WHERE mp.is_deleted = false - GROUP BY - mp.project_id, - mp.project_name, - mp.project_description, - ppll.project_team_members, - mp.ecapris_subproject_id, +GROUP BY + mp.project_id, + mp.project_name, + mp.project_description, + ppll.project_team_members, + mp.ecapris_subproject_id, mp.date_added, mp.is_deleted, me.entity_name, diff --git a/moped-database/views/uniform_features.sql b/moped-database/views/uniform_features.sql index cf70aa7708..7aa97796a1 100644 --- a/moped-database/views/uniform_features.sql +++ b/moped-database/views/uniform_features.sql @@ -1,10 +1,20 @@ -- latest version 1700149643859_add_audit_fields_to_unified_features_view CREATE OR REPLACE VIEW uniform_features AS - SELECT feature_signals.id, +SELECT + feature_signals.id, feature_signals.component_id, 'feature_signals'::text AS "table", - json_build_object('signal_id', feature_signals.signal_id, 'knack_id', feature_signals.knack_id, 'location_name', feature_signals.location_name, 'signal_type', feature_signals.signal_type) AS attributes, + json_build_object( + 'signal_id', + feature_signals.signal_id, + 'knack_id', + feature_signals.knack_id, + 'location_name', + feature_signals.location_name, + 'signal_type', + feature_signals.signal_type + ) AS attributes, feature_signals.geography, districts.council_districts, NULL::integer AS length_feet, @@ -12,17 +22,36 @@ CREATE OR REPLACE VIEW uniform_features AS feature_signals.updated_at, feature_signals.created_by_user_id, feature_signals.updated_by_user_id - FROM feature_signals - LEFT JOIN ( SELECT d.feature_id, - array_agg(d.council_district_id) AS council_districts - FROM features_council_districts d - GROUP BY d.feature_id) districts ON districts.feature_id = feature_signals.id - WHERE feature_signals.is_deleted = false +FROM feature_signals +LEFT JOIN ( + SELECT + d.feature_id, + array_agg(d.council_district_id) AS council_districts + FROM features_council_districts AS d + GROUP BY d.feature_id +) AS districts ON feature_signals.id = districts.feature_id +WHERE feature_signals.is_deleted = FALSE UNION ALL - SELECT feature_street_segments.id, +SELECT + feature_street_segments.id, feature_street_segments.component_id, 'feature_street_segments'::text AS "table", - json_build_object('ctn_segment_id', feature_street_segments.ctn_segment_id, 'from_address_min', feature_street_segments.from_address_min, 'to_address_max', feature_street_segments.to_address_max, 'full_street_name', feature_street_segments.full_street_name, 'line_type', feature_street_segments.line_type, 'symbol', feature_street_segments.symbol, 'source_layer', feature_street_segments.source_layer) AS attributes, + json_build_object( + 'ctn_segment_id', + feature_street_segments.ctn_segment_id, + 'from_address_min', + feature_street_segments.from_address_min, + 'to_address_max', + feature_street_segments.to_address_max, + 'full_street_name', + feature_street_segments.full_street_name, + 'line_type', + feature_street_segments.line_type, + 'symbol', + feature_street_segments.symbol, + 'source_layer', + feature_street_segments.source_layer + ) AS attributes, feature_street_segments.geography, districts.council_districts, feature_street_segments.length_feet, @@ -30,17 +59,23 @@ UNION ALL feature_street_segments.updated_at, feature_street_segments.created_by_user_id, feature_street_segments.updated_by_user_id - FROM feature_street_segments - LEFT JOIN ( SELECT d.feature_id, - array_agg(d.council_district_id) AS council_districts - FROM features_council_districts d - GROUP BY d.feature_id) districts ON districts.feature_id = feature_street_segments.id - WHERE feature_street_segments.is_deleted = false +FROM feature_street_segments +LEFT JOIN ( + SELECT + d.feature_id, + array_agg(d.council_district_id) AS council_districts + FROM features_council_districts AS d + GROUP BY d.feature_id +) AS districts ON feature_street_segments.id = districts.feature_id +WHERE feature_street_segments.is_deleted = FALSE UNION ALL - SELECT feature_intersections.id, +SELECT + feature_intersections.id, feature_intersections.component_id, 'feature_intersections'::text AS "table", - json_build_object('intersection_id', feature_intersections.intersection_id, 'source_layer', feature_intersections.source_layer) AS attributes, + json_build_object( + 'intersection_id', feature_intersections.intersection_id, 'source_layer', feature_intersections.source_layer + ) AS attributes, feature_intersections.geography, districts.council_districts, NULL::integer AS length_feet, @@ -48,14 +83,18 @@ UNION ALL feature_intersections.updated_at, feature_intersections.created_by_user_id, feature_intersections.updated_by_user_id - FROM feature_intersections - LEFT JOIN ( SELECT d.feature_id, - array_agg(d.council_district_id) AS council_districts - FROM features_council_districts d - GROUP BY d.feature_id) districts ON districts.feature_id = feature_intersections.id - WHERE feature_intersections.is_deleted = false +FROM feature_intersections +LEFT JOIN ( + SELECT + d.feature_id, + array_agg(d.council_district_id) AS council_districts + FROM features_council_districts AS d + GROUP BY d.feature_id +) AS districts ON feature_intersections.id = districts.feature_id +WHERE feature_intersections.is_deleted = FALSE UNION ALL - SELECT feature_drawn_points.id, +SELECT + feature_drawn_points.id, feature_drawn_points.component_id, 'feature_drawn_points'::text AS "table", NULL::json AS attributes, @@ -66,14 +105,18 @@ UNION ALL feature_drawn_points.updated_at, feature_drawn_points.created_by_user_id, feature_drawn_points.updated_by_user_id - FROM feature_drawn_points - LEFT JOIN ( SELECT d.feature_id, - array_agg(d.council_district_id) AS council_districts - FROM features_council_districts d - GROUP BY d.feature_id) districts ON districts.feature_id = feature_drawn_points.id - WHERE feature_drawn_points.is_deleted = false +FROM feature_drawn_points +LEFT JOIN ( + SELECT + d.feature_id, + array_agg(d.council_district_id) AS council_districts + FROM features_council_districts AS d + GROUP BY d.feature_id +) AS districts ON feature_drawn_points.id = districts.feature_id +WHERE feature_drawn_points.is_deleted = FALSE UNION ALL - SELECT feature_drawn_lines.id, +SELECT + feature_drawn_lines.id, feature_drawn_lines.component_id, 'feature_drawn_lines'::text AS "table", NULL::json AS attributes, @@ -84,11 +127,14 @@ UNION ALL feature_drawn_lines.updated_at, feature_drawn_lines.created_by_user_id, feature_drawn_lines.updated_by_user_id - FROM feature_drawn_lines - LEFT JOIN ( SELECT d.feature_id, - array_agg(d.council_district_id) AS council_districts - FROM features_council_districts d - GROUP BY d.feature_id) districts ON districts.feature_id = feature_drawn_lines.id - WHERE feature_drawn_lines.is_deleted = false; +FROM feature_drawn_lines +LEFT JOIN ( + SELECT + d.feature_id, + array_agg(d.council_district_id) AS council_districts + FROM features_council_districts AS d + GROUP BY d.feature_id +) AS districts ON feature_drawn_lines.id = districts.feature_id +WHERE feature_drawn_lines.is_deleted = FALSE; COMMENT ON VIEW uniform_features IS 'This view unifies various geographical feature data from multiple tables such as signals, street segments, intersections, drawn points, and lines. It provides a view of these features along with their attributes, geographic details, and council district associations.'; From f687c7497a8f99ea86b679ee809e15395b2136fa Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 7 Feb 2024 12:38:19 -0600 Subject: [PATCH 30/76] Improve logging --- moped-etl/data-tracker-sync/data_tracker_sync.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/moped-etl/data-tracker-sync/data_tracker_sync.py b/moped-etl/data-tracker-sync/data_tracker_sync.py index 6aaffde8d7..97e477b6e0 100644 --- a/moped-etl/data-tracker-sync/data_tracker_sync.py +++ b/moped-etl/data-tracker-sync/data_tracker_sync.py @@ -48,7 +48,9 @@ def find_unsynced_moped_projects(is_test=False): data = make_hasura_request(query=GET_UNSYNCED_PROJECTS) unsynced_projects = data["moped_project"] - logger.info(f"Found {len(unsynced_projects)} unsynced projects") + logger.info( + f"Found {len(unsynced_projects)} unsynced projects to create in Data Tracker" + ) logger.debug(f"Found unsynced projects: {unsynced_projects}") return unsynced_projects @@ -138,7 +140,9 @@ def find_synced_moped_projects(last_run_date, is_test=False): ) synced_projects = data["moped_project"] - logger.info(f"Found {len(synced_projects)} synced projects") + logger.info( + f"Found {len(synced_projects)} synced projects to update in Data Tracker" + ) logger.debug(f"Found synced projects: {synced_projects}") return synced_projects @@ -179,6 +183,7 @@ def main(args): unsynced_moped_projects = find_unsynced_moped_projects(is_test=args.test) # Create a Knack project for each unsynced Moped project + logger.info(f"Creating Knack records...") created_knack_records = [] for project in unsynced_moped_projects: moped_project_id = project["project_id"] @@ -204,6 +209,7 @@ def main(args): ) # Update synced Moped projects in Data Tracker and skip those just created + logger.info(f"Updating...") updated_knack_records = [] updates_to_skip = [record["moped_project_id"] for record in created_knack_records] From 506fad12a60210ab33fdbf28d802b0a048c71c1f Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 7 Feb 2024 13:04:47 -0600 Subject: [PATCH 31/76] EOF LF --- .sqlfluff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sqlfluff b/.sqlfluff index ef4951a446..b0e6521861 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -68,4 +68,4 @@ extended_capitalisation_policy = consistent [sqlfluff:rules:capitalisation.literals] capitalisation_policy = consistent [sqlfluff:rules:capitalisation.types] -extended_capitalisation_policy = consistent \ No newline at end of file +extended_capitalisation_policy = consistent From beebbee9f5999996d76bea055dd60267db535c0d Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 7 Feb 2024 14:25:20 -0600 Subject: [PATCH 32/76] format view --- .../views/component_arcgis_online_view.sql | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/moped-database/views/component_arcgis_online_view.sql b/moped-database/views/component_arcgis_online_view.sql index 8a5e6d1896..38cabc5423 100644 --- a/moped-database/views/component_arcgis_online_view.sql +++ b/moped-database/views/component_arcgis_online_view.sql @@ -12,7 +12,8 @@ CREATE OR REPLACE VIEW component_arcgis_online_view AS ( GROUP BY project_component_id ), -council_districts AS ( + + council_districts AS ( -- group council districts by project component id SELECT component_id AS project_component_id, @@ -28,7 +29,8 @@ council_districts AS ( GROUP BY component_id ), -comp_geography AS ( + + comp_geography AS ( -- group feature properties by project component ID SELECT component_id AS project_component_id, @@ -117,7 +119,8 @@ comp_geography AS ( GROUP BY component_id ), -subcomponents AS ( + + subcomponents AS ( -- group subcomponents by project component id SELECT project_component_id, @@ -129,7 +132,8 @@ subcomponents AS ( GROUP BY project_component_id ), -component_tags AS ( + + component_tags AS ( -- group project component tags by project component id SELECT project_component_id, @@ -141,6 +145,7 @@ component_tags AS ( GROUP BY project_component_id ) + SELECT mpc.project_id, comp_geography.project_component_id, @@ -151,7 +156,7 @@ component_tags AS ( comp_geography.signal_ids, council_districts.council_districts, 'placeholder text' AS council_districts_searchable, - NOT coalesce (council_districts IS NULL OR council_districts = '', FALSE) AS is_within_city_limits, + NOT coalesce(council_districts IS NULL OR council_districts = '', FALSE) AS is_within_city_limits, comp_geography.length_feet_total, round((comp_geography.length_feet_total / 5280::numeric), 2) AS length_miles_total, mc.component_name, @@ -546,7 +551,8 @@ CREATE OR REPLACE VIEW component_arcgis_online_view AS ( GROUP BY project_component_id ), -council_districts AS ( + + council_districts AS ( -- group council districts by project component id SELECT component_id AS project_component_id, @@ -562,7 +568,8 @@ council_districts AS ( GROUP BY component_id ), -comp_geography AS ( + + comp_geography AS ( -- group feature properties by project component ID SELECT component_id AS project_component_id, @@ -651,7 +658,8 @@ comp_geography AS ( GROUP BY component_id ), -subcomponents AS ( + + subcomponents AS ( -- group subcomponents by project component id SELECT project_component_id, @@ -663,7 +671,8 @@ subcomponents AS ( GROUP BY project_component_id ), -component_tags AS ( + + component_tags AS ( -- group project component tags by project component id SELECT project_component_id, @@ -675,6 +684,7 @@ component_tags AS ( GROUP BY project_component_id ) + SELECT mpc.project_id, comp_geography.project_component_id, @@ -685,7 +695,7 @@ component_tags AS ( comp_geography.signal_ids, council_districts.council_districts, 'placeholder text' AS council_districts_searchable, - NOT coalesce (council_districts IS NULL OR council_districts = '', FALSE) AS is_within_city_limits, + NOT coalesce(council_districts IS NULL OR council_districts = '', FALSE) AS is_within_city_limits, comp_geography.length_feet_total, round((comp_geography.length_feet_total / 5280::numeric), 2) AS length_miles_total, mc.component_name, From 71438056ca71a1e7563d29052d238bb975972fba Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 7 Feb 2024 14:35:57 -0600 Subject: [PATCH 33/76] Bring this file back into sync with the DB --- .../views/component_arcgis_online_view.sql | 963 ++++-------------- 1 file changed, 200 insertions(+), 763 deletions(-) diff --git a/moped-database/views/component_arcgis_online_view.sql b/moped-database/views/component_arcgis_online_view.sql index 38cabc5423..a4180ad908 100644 --- a/moped-database/views/component_arcgis_online_view.sql +++ b/moped-database/views/component_arcgis_online_view.sql @@ -1,780 +1,217 @@ --- current_version: 1706897389736_fix_sub_comp_date_dash -CREATE OR REPLACE VIEW component_arcgis_online_view AS ( - WITH work_types AS ( - -- group work types by project component id - SELECT - project_component_id, - string_agg(mwt.name, ', ') AS work_types - FROM - moped_proj_component_work_types AS mpcwt - LEFT JOIN moped_work_types AS mwt ON mpcwt.work_type_id = mwt.id - WHERE mpcwt.is_deleted = FALSE - GROUP BY - project_component_id - ), - - council_districts AS ( - -- group council districts by project component id - SELECT - component_id AS project_component_id, - string_agg( - DISTINCT council_district_id::text, - ', ' - ) AS council_districts - FROM - features_council_districts - LEFT JOIN features ON features_council_districts.feature_id = features.id - WHERE - features.is_deleted = FALSE - GROUP BY - component_id - ), - - comp_geography AS ( - -- group feature properties by project component ID - SELECT - component_id AS project_component_id, - string_agg(DISTINCT id::text, ', ') AS feature_ids, - st_asgeojson( - st_union( - array_agg(geography) - ) - )::json AS "geometry", - st_asgeojson( - st_union( - array_agg(line_geography) - ) - )::json AS "line_geometry", - string_agg(DISTINCT signal_id::text, ', ') AS signal_ids, - sum(length_feet) AS length_feet_total - FROM - ( - -- union all features - SELECT - id, - feature_signals.component_id, - feature_signals.geography::geometry, - st_exteriorring( - st_buffer(feature_signals.geography, 7)::geometry - ) AS line_geography, - feature_signals.signal_id, - NULL AS length_feet - FROM - feature_signals - WHERE - feature_signals.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_street_segments.component_id, - feature_street_segments.geography::geometry, - feature_street_segments.geography::geometry AS line_geography, - NULL AS signal_id, - length_feet - FROM - feature_street_segments - WHERE - feature_street_segments.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_intersections.component_id, - feature_intersections.geography::geometry, - st_exteriorring( - st_buffer(feature_intersections.geography, 7)::geometry - ) AS line_geography, - NULL AS signal_id, - NULL AS length_feet - FROM - feature_intersections - WHERE - feature_intersections.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_drawn_points.component_id, - feature_drawn_points.geography::geometry, - st_exteriorring( - st_buffer(feature_drawn_points.geography, 7)::geometry - ) AS line_geography, - NULL AS signal_id, - NULL AS length_feet - FROM - feature_drawn_points - WHERE - feature_drawn_points.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_drawn_lines.component_id, - feature_drawn_lines.geography::geometry, - feature_drawn_lines.geography::geometry AS line_geography, - NULL AS signal_id, - length_feet - FROM - feature_drawn_lines - WHERE - feature_drawn_lines.is_deleted = FALSE - ) AS feature_union - GROUP BY - component_id - ), - - subcomponents AS ( - -- group subcomponents by project component id - SELECT - project_component_id, - string_agg(ms.subcomponent_name, ', ') AS subcomponents - FROM - moped_proj_components_subcomponents AS mpcs - LEFT JOIN moped_subcomponents AS ms ON mpcs.subcomponent_id = ms.subcomponent_id - WHERE mpcs.is_deleted = FALSE - GROUP BY - project_component_id - ), - - component_tags AS ( - -- group project component tags by project component id - SELECT - project_component_id, - string_agg(mct.type || ' - ' || mct.name, ', ') AS component_tags - FROM - moped_proj_component_tags AS mpct - LEFT JOIN moped_component_tags AS mct ON mpct.component_tag_id = mct.id - WHERE mpct.is_deleted = FALSE - GROUP BY - project_component_id - ) - - SELECT - mpc.project_id, - comp_geography.project_component_id, - comp_geography.feature_ids, - mpc.component_id, - comp_geography.geometry, - comp_geography.line_geometry, - comp_geography.signal_ids, - council_districts.council_districts, - 'placeholder text' AS council_districts_searchable, - NOT coalesce(council_districts IS NULL OR council_districts = '', FALSE) AS is_within_city_limits, - comp_geography.length_feet_total, - round((comp_geography.length_feet_total / 5280::numeric), 2) AS length_miles_total, - mc.component_name, - mc.component_subtype, - mc.component_name_full, - 'placeholder text' AS component_categories, - subcomponents.subcomponents AS component_subcomponents, - work_types.work_types AS component_work_types, - component_tags.component_tags, - mpc.description AS component_description, - mpc.interim_project_component_id, - mpc.completion_date, - coalesce(mpc.completion_date, substantial_completion_date) AS substantial_completion_date, - '2024-01-01T00:00:00-06:00' AS substantial_completion_date_estimated, - mpc.srts_id, - mpc.location_description AS component_location_description, - plv.project_name, - 'placeholder text' AS project_name_descriptor, - 'placeholder text' AS project_name_with_descriptor, - plv.project_description, - plv.ecapris_subproject_id, - plv.project_website, - plv.updated_at AS project_updated_at, - mpc.phase_id AS component_phase_id, - mph.phase_name AS component_phase_name, - mph.phase_name_simple AS component_phase_name_simple, - current_phase.phase_id AS project_phase_id, - current_phase.phase_name AS project_phase_name, - current_phase.phase_name_simple AS project_phase_name_simple, - coalesce(mph.phase_name, current_phase.phase_name) AS current_phase_name, - coalesce(mph.phase_name_simple, current_phase.phase_name_simple) AS current_phase_name_simple, - plv.project_team_members, - plv.project_sponsor, - plv.project_lead, - plv.public_process_status, - plv.interim_project_id, - plv.project_partners, - plv.task_order_names, - plv.funding_source_name, - plv.type_name, - plv.project_status_update, - plv.project_status_update_date_created, - plv.construction_start_date, - plv.completion_end_date, - plv.project_inspector, - plv.project_designer, - plv.project_tags, - plv.workgroup_contractors, - plv.contract_numbers, - plv.parent_project_id, - plv.parent_project_name, - plv.parent_project_url, - 'placeholder text' AS parent_project_name_with_descriptor, - 'placeholder text' AS related_project_ids, - 'placeholder text' AS related_project_ids_searchable, - plv.knack_project_id AS knack_data_tracker_project_record_id, - plv.project_url, - plv.project_url || '?tab=map&project_component_id=' || mpc.project_component_id::text AS component_url, - plv.project_development_status, - plv.project_development_status_date, - plv.project_development_status_date_calendar_year, - plv.project_development_status_date_calendar_year_month, - plv.project_development_status_date_calendar_year_month_numeric, - plv.project_development_status_date_calendar_year_quarter, - plv.project_development_status_date_fiscal_year, - plv.project_development_status_date_fiscal_year_quarter, - added_by AS project_added_by - FROM - moped_proj_components AS mpc - LEFT JOIN comp_geography ON mpc.project_component_id = comp_geography.project_component_id - LEFT JOIN council_districts ON mpc.project_component_id = council_districts.project_component_id - LEFT JOIN subcomponents ON mpc.project_component_id = subcomponents.project_component_id - LEFT JOIN work_types ON mpc.project_component_id = work_types.project_component_id - LEFT JOIN component_tags ON mpc.project_component_id = component_tags.project_component_id - LEFT JOIN project_list_view AS plv ON mpc.project_id = plv.project_id - LEFT JOIN current_phase_view AS current_phase ON mpc.project_id = current_phase.project_id - LEFT JOIN moped_phases AS mph ON mpc.phase_id = mph.phase_id - LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id - WHERE - mpc.is_deleted = FALSE - AND plv.is_deleted = FALSE -); - -DROP VIEW project_list_view CASCADE; - -CREATE OR REPLACE VIEW public.project_list_view -AS WITH project_person_list_lookup AS ( - SELECT - mpp.project_id, - string_agg( - DISTINCT concat(mu.first_name, ' ', mu.last_name, ':', mpr.project_role_name), ','::text - ) AS project_team_members - FROM moped_proj_personnel AS mpp - INNER JOIN moped_users AS mu ON mpp.user_id = mu.user_id - INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id - INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id - WHERE mpp.is_deleted = FALSE - AND mppr.is_deleted = FALSE - GROUP BY mpp.project_id -), - -funding_sources_lookup AS ( +CREATE OR REPLACE VIEW component_arcgis_online_view AS +WITH work_types AS ( SELECT - mpf_1.project_id, - string_agg(mfs.funding_source_name, ', '::text) AS funding_source_name - FROM moped_proj_funding AS mpf_1 - LEFT JOIN moped_fund_sources AS mfs ON mpf_1.funding_source_id = mfs.funding_source_id - WHERE mpf_1.is_deleted = FALSE - GROUP BY mpf_1.project_id + mpcwt.project_component_id, + string_agg(mwt.name, ', '::text) AS work_types + FROM ( + moped_proj_component_work_types AS mpcwt + LEFT JOIN moped_work_types AS mwt ON ((mpcwt.work_type_id = mwt.id)) + ) + WHERE (mpcwt.is_deleted = false) + GROUP BY mpcwt.project_component_id ), -project_type_lookup AS ( +council_districts AS ( SELECT - mpt.project_id, - string_agg(mt.type_name, ', '::text) AS type_name - FROM moped_project_types AS mpt - LEFT JOIN moped_types AS mt ON mpt.project_type_id = mt.type_id AND mpt.is_deleted = FALSE - GROUP BY mpt.project_id + features.component_id AS project_component_id, + string_agg(DISTINCT (features_council_districts.council_district_id)::text, ', '::text) AS council_districts + FROM ( + features_council_districts + LEFT JOIN features ON ((features_council_districts.feature_id = features.id)) + ) + WHERE (features.is_deleted = false) + GROUP BY features.component_id ), -child_project_lookup AS ( +comp_geography AS ( SELECT - jsonb_agg(children.project_id) AS children_project_ids, - children.parent_project_id AS parent_id - FROM moped_project AS children - INNER JOIN moped_project AS parent ON (children.parent_project_id = parent.project_id) - WHERE children.is_deleted = FALSE - GROUP BY parent_id + feature_union.component_id AS project_component_id, + string_agg(DISTINCT (feature_union.id)::text, ', '::text) AS feature_ids, + (st_asgeojson(st_union(array_agg(feature_union.geography))))::json AS geometry, + (st_asgeojson(st_union(array_agg(feature_union.line_geography))))::json AS line_geometry, + string_agg(DISTINCT (feature_union.signal_id)::text, ', '::text) AS signal_ids, + sum(feature_union.length_feet) AS length_feet_total + FROM ( + SELECT + feature_signals.id, + feature_signals.component_id, + (feature_signals.geography)::geometry AS geography, + st_exteriorring((st_buffer(feature_signals.geography, (7)::double precision))::geometry) AS line_geography, + feature_signals.signal_id, + null::integer AS length_feet + FROM feature_signals + WHERE (feature_signals.is_deleted = false) + UNION ALL + SELECT + feature_street_segments.id, + feature_street_segments.component_id, + (feature_street_segments.geography)::geometry AS geography, + (feature_street_segments.geography)::geometry AS line_geography, + null::integer AS signal_id, + feature_street_segments.length_feet + FROM feature_street_segments + WHERE (feature_street_segments.is_deleted = false) + UNION ALL + SELECT + feature_intersections.id, + feature_intersections.component_id, + (feature_intersections.geography)::geometry AS geography, + st_exteriorring( + (st_buffer(feature_intersections.geography, (7)::double precision))::geometry + ) AS line_geography, + null::integer AS signal_id, + null::integer AS length_feet + FROM feature_intersections + WHERE (feature_intersections.is_deleted = false) + UNION ALL + SELECT + feature_drawn_points.id, + feature_drawn_points.component_id, + (feature_drawn_points.geography)::geometry AS geography, + st_exteriorring( + (st_buffer(feature_drawn_points.geography, (7)::double precision))::geometry + ) AS line_geography, + null::integer AS signal_id, + null::integer AS length_feet + FROM feature_drawn_points + WHERE (feature_drawn_points.is_deleted = false) + UNION ALL + SELECT + feature_drawn_lines.id, + feature_drawn_lines.component_id, + (feature_drawn_lines.geography)::geometry AS geography, + (feature_drawn_lines.geography)::geometry AS line_geography, + null::integer AS signal_id, + feature_drawn_lines.length_feet + FROM feature_drawn_lines + WHERE (feature_drawn_lines.is_deleted = false) + ) AS feature_union + GROUP BY feature_union.component_id ), -work_activities AS ( +subcomponents AS ( SELECT - project_id, - string_agg( - task_order_objects.task_order_object ->> 'display_name'::text, - ', '::text - ) AS task_order_names, - string_agg( - task_order_objects.task_order_object ->> 'task_order'::text, - ', '::text - ) AS task_order_names_short, - jsonb_agg(DISTINCT task_order_objects.task_order_object) FILTER ( - WHERE task_order_objects.task_order_object IS NOT NULL - ) AS task_orders, - string_agg( - DISTINCT mpwa.workgroup_contractor, - ', '::text - ) AS workgroup_contractors, - string_agg( - mpwa.contract_number, - ', '::text - ) AS contract_numbers - FROM moped_proj_work_activity AS mpwa - LEFT JOIN LATERAL jsonb_array_elements(mpwa.task_orders) AS task_order_objects (task_order_object) - ON TRUE WHERE 1 = 1 - AND mpwa.is_deleted = FALSE - GROUP BY - mpwa.project_id + mpcs.project_component_id, + string_agg(ms.subcomponent_name, ', '::text) AS subcomponents + FROM ( + moped_proj_components_subcomponents AS mpcs + LEFT JOIN moped_subcomponents AS ms ON ((mpcs.subcomponent_id = ms.subcomponent_id)) + ) + WHERE (mpcs.is_deleted = false) + GROUP BY mpcs.project_component_id ), -moped_proj_components_subtypes AS ( +component_tags AS ( SELECT - mpc.project_id, - string_agg(DISTINCT mc.component_name_full, ', '::text) AS components - FROM moped_proj_components AS mpc - LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id - WHERE mpc.is_deleted = FALSE - GROUP BY mpc.project_id + mpct.project_component_id, + string_agg(((mct.type || ' - '::text) || mct.name), ', '::text) AS component_tags + FROM ( + moped_proj_component_tags AS mpct + LEFT JOIN moped_component_tags AS mct ON ((mpct.component_tag_id = mct.id)) + ) + WHERE (mpct.is_deleted = false) + GROUP BY mpct.project_component_id ) SELECT - mp.project_id, - mp.project_name, - mp.project_description, - mp.ecapris_subproject_id, - mp.project_website, - mp.date_added, - mp.is_deleted, - mp.updated_at, - current_phase.phase_name AS current_phase, - current_phase.phase_key AS current_phase_key, - current_phase.phase_name_simple AS current_phase_simple, - ppll.project_team_members, - me.entity_name AS project_sponsor, - mel.entity_name AS project_lead, - mpps.name AS public_process_status, - mp.interim_project_id, - mp.parent_project_id, - mp.knack_project_id, - 'https://mobility.austin.gov/moped/projects/' || mp.project_id::text AS project_url, - 'https://mobility.austin.gov/moped/projects/' || mp.parent_project_id::text AS parent_project_url, - proj_status_update.project_note AS project_status_update, - proj_status_update.date_created AS project_status_update_date_created, - work_activities.workgroup_contractors, - work_activities.contract_numbers, - work_activities.task_order_names, - work_activities.task_order_names_short, - work_activities.task_orders, - 'placeholder text' AS project_development_status, - '2024-01-01T00:00:00-06:00' AS project_development_status_date, - 9999 AS project_development_status_date_calendar_year, - 'placeholder text' AS project_development_status_date_calendar_year_month, - 'placeholder text' AS project_development_status_date_calendar_year_month_numeric, - 'placeholder text' AS project_development_status_date_calendar_year_quarter, - 999 AS project_development_status_date_fiscal_year, - 'placeholder text' AS project_development_status_date_fiscal_year_quarter, + mpc.project_id, + comp_geography.project_component_id, + comp_geography.feature_ids, + mpc.component_id, + comp_geography.geometry, + comp_geography.line_geometry, + comp_geography.signal_ids, + council_districts.council_districts, + 'placeholder text'::text AS council_districts_searchable, ( - SELECT project_name - FROM moped_project - WHERE project_id = mp.parent_project_id - ) AS parent_project_name, - cpl.children_project_ids, - string_agg(DISTINCT me2.entity_name, ', '::text) AS project_partners, - ( - SELECT - json_agg( - json_build_object( - 'signal_id', - feature_signals.signal_id, - 'knack_id', - feature_signals.knack_id, - 'location_name', - feature_signals.location_name, - 'signal_type', - feature_signals.signal_type, - 'id', - feature_signals.id - ) - ) - FROM moped_proj_components AS components - LEFT JOIN feature_signals - ON (components.project_component_id = feature_signals.component_id) - WHERE TRUE - AND components.is_deleted = FALSE - AND components.project_id = mp.project_id - AND feature_signals.signal_id IS NOT NULL - AND feature_signals.is_deleted = FALSE - ) AS project_feature, - fsl.funding_source_name, - ptl.type_name, - ( -- get the date of the construction phase with the earliest start date - SELECT min(phases.phase_start) - FROM moped_proj_phases AS phases - WHERE TRUE - AND phases.project_id = mp.project_id - AND phases.phase_id = 9 -- phase_id 9 is construction - AND phases.is_deleted = FALSE - ) AS construction_start_date, - ( -- get the date of the completion phase with the latest end date - SELECT max(phases.phase_end) - FROM moped_proj_phases AS phases - WHERE TRUE - AND phases.project_id = mp.project_id - AND phases.phase_id = 11 -- phase_id 11 is complete - AND phases.is_deleted = FALSE - ) AS completion_end_date, - ( -- get the earliest confirmed phase_start or phase_end with a simple phase of 'Complete' - SELECT min(min_confirmed_date) - FROM ( - -- earliest confirmed phase start - SELECT min(phases.phase_start) AS min_confirmed_date - FROM - moped_proj_phases AS phases - LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id - WHERE - TRUE - AND phases.phase_start IS NOT NULL - AND phases.is_phase_start_confirmed = TRUE - AND phases.project_id = mp.project_id - AND moped_phases.phase_name_simple = 'Complete' - AND phases.is_deleted = FALSE - UNION ALL - -- earliest confirmed phase end - SELECT min(phases.phase_end) AS min_confirmed_date - FROM - moped_proj_phases AS phases - LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id - WHERE - TRUE - AND phases.phase_end IS NOT NULL - AND phases.is_phase_end_confirmed = TRUE - AND phases.project_id = mp.project_id - AND moped_phases.phase_name_simple = 'Complete' - AND phases.is_deleted = FALSE - ) AS min_confirmed_dates - ) AS substantial_completion_date, - ( -- get me a list of the inspectors for this project - SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg - FROM moped_proj_personnel AS mpp - INNER JOIN moped_users AS users ON mpp.user_id = users.user_id - INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id - INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id - WHERE 1 = 1 - AND mpr.project_role_name = 'Inspector'::text - AND mpp.is_deleted = FALSE - AND mppr.is_deleted = FALSE - AND mpp.project_id = mp.project_id - GROUP BY mpp.project_id - ) AS project_inspector, - ( -- get me a list of the designers for this project - SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg - FROM moped_proj_personnel AS mpp - INNER JOIN moped_users AS users ON mpp.user_id = users.user_id - INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id - INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id - WHERE 1 = 1 - AND mpr.project_role_name = 'Designer'::text - AND mpp.is_deleted = FALSE - AND mppr.is_deleted = FALSE - AND mpp.project_id = mp.project_id - GROUP BY mpp.project_id - ) AS project_designer, - ( -- get me all of the tags added to a project - SELECT string_agg(tags.name, ', '::text) AS string_agg - FROM moped_proj_tags AS ptags - INNER JOIN moped_tags AS tags ON ptags.tag_id = tags.id - WHERE 1 = 1 - AND ptags.is_deleted = FALSE - AND ptags.project_id = mp.project_id - GROUP BY ptags.project_id - ) AS project_tags, - concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, - mpcs.components -FROM moped_project AS mp -LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id -LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id -LEFT JOIN project_type_lookup AS ptl ON mp.project_id = ptl.project_id -LEFT JOIN moped_entity AS me ON mp.project_sponsor = me.entity_id -LEFT JOIN moped_entity AS mel ON mp.project_lead_id = mel.entity_id -LEFT JOIN moped_proj_partners AS mpp2 ON mp.project_id = mpp2.project_id AND mpp2.is_deleted = FALSE -LEFT JOIN moped_entity AS me2 ON mpp2.entity_id = me2.entity_id -LEFT JOIN work_activities ON mp.project_id = work_activities.project_id -LEFT JOIN moped_users AS added_by_user ON mp.added_by = added_by_user.user_id -LEFT JOIN current_phase_view AS current_phase ON mp.project_id = current_phase.project_id -LEFT JOIN moped_public_process_statuses AS mpps ON mp.public_process_status_id = mpps.id -LEFT JOIN child_project_lookup AS cpl ON mp.project_id = cpl.parent_id -LEFT JOIN moped_proj_components_subtypes AS mpcs ON mp.project_id = mpcs.project_id -LEFT JOIN LATERAL - ( - SELECT - mpn.project_note, - mpn.date_created - FROM moped_proj_notes AS mpn - WHERE mpn.project_id = mp.project_id AND mpn.project_note_type = 2 AND mpn.is_deleted = FALSE - ORDER BY mpn.date_created DESC - LIMIT 1 - ) AS proj_status_update ON TRUE -WHERE - mp.is_deleted = FALSE -GROUP BY - mp.project_id, - mp.project_name, - mp.project_description, - ppll.project_team_members, - mp.ecapris_subproject_id, - mp.date_added, - mp.is_deleted, - me.entity_name, - mel.entity_name, - mp.updated_at, - mp.interim_project_id, - mp.parent_project_id, - mp.knack_project_id, - current_phase.phase_name, - current_phase.phase_key, - current_phase.phase_name_simple, - ptl.type_name, - mpcs.components, - fsl.funding_source_name, - added_by_user.first_name, - added_by_user.last_name, - mpps.name, - cpl.children_project_ids, - proj_status_update.project_note, - proj_status_update.date_created, - work_activities.workgroup_contractors, - work_activities.contract_numbers, - work_activities.task_order_names, - work_activities.task_order_names_short, - work_activities.task_orders; - -CREATE OR REPLACE VIEW component_arcgis_online_view AS ( - WITH work_types AS ( - -- group work types by project component id - SELECT - project_component_id, - string_agg(mwt.name, ', ') AS work_types - FROM - moped_proj_component_work_types AS mpcwt - LEFT JOIN moped_work_types AS mwt ON mpcwt.work_type_id = mwt.id - WHERE mpcwt.is_deleted = FALSE - GROUP BY - project_component_id - ), - - council_districts AS ( - -- group council districts by project component id - SELECT - component_id AS project_component_id, - string_agg( - DISTINCT council_district_id::text, - ', ' - ) AS council_districts - FROM - features_council_districts - LEFT JOIN features ON features_council_districts.feature_id = features.id - WHERE - features.is_deleted = FALSE - GROUP BY - component_id - ), - - comp_geography AS ( - -- group feature properties by project component ID - SELECT - component_id AS project_component_id, - string_agg(DISTINCT id::text, ', ') AS feature_ids, - st_asgeojson( - st_union( - array_agg(geography) - ) - )::json AS "geometry", - st_asgeojson( - st_union( - array_agg(line_geography) - ) - )::json AS "line_geometry", - string_agg(DISTINCT signal_id::text, ', ') AS signal_ids, - sum(length_feet) AS length_feet_total - FROM - ( - -- union all features - SELECT - id, - feature_signals.component_id, - feature_signals.geography::geometry, - st_exteriorring( - st_buffer(feature_signals.geography, 7)::geometry - ) AS line_geography, - feature_signals.signal_id, - NULL AS length_feet - FROM - feature_signals - WHERE - feature_signals.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_street_segments.component_id, - feature_street_segments.geography::geometry, - feature_street_segments.geography::geometry AS line_geography, - NULL AS signal_id, - length_feet - FROM - feature_street_segments - WHERE - feature_street_segments.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_intersections.component_id, - feature_intersections.geography::geometry, - st_exteriorring( - st_buffer(feature_intersections.geography, 7)::geometry - ) AS line_geography, - NULL AS signal_id, - NULL AS length_feet - FROM - feature_intersections - WHERE - feature_intersections.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_drawn_points.component_id, - feature_drawn_points.geography::geometry, - st_exteriorring( - st_buffer(feature_drawn_points.geography, 7)::geometry - ) AS line_geography, - NULL AS signal_id, - NULL AS length_feet - FROM - feature_drawn_points - WHERE - feature_drawn_points.is_deleted = FALSE - UNION ALL - SELECT - id, - feature_drawn_lines.component_id, - feature_drawn_lines.geography::geometry, - feature_drawn_lines.geography::geometry AS line_geography, - NULL AS signal_id, - length_feet - FROM - feature_drawn_lines - WHERE - feature_drawn_lines.is_deleted = FALSE - ) AS feature_union - GROUP BY - component_id - ), - - subcomponents AS ( - -- group subcomponents by project component id - SELECT - project_component_id, - string_agg(ms.subcomponent_name, ', ') AS subcomponents - FROM - moped_proj_components_subcomponents AS mpcs - LEFT JOIN moped_subcomponents AS ms ON mpcs.subcomponent_id = ms.subcomponent_id - WHERE mpcs.is_deleted = FALSE - GROUP BY - project_component_id - ), - - component_tags AS ( - -- group project component tags by project component id - SELECT - project_component_id, - string_agg(mct.type || ' - ' || mct.name, ', ') AS component_tags - FROM - moped_proj_component_tags AS mpct - LEFT JOIN moped_component_tags AS mct ON mpct.component_tag_id = mct.id - WHERE mpct.is_deleted = FALSE - GROUP BY - project_component_id - ) - - SELECT - mpc.project_id, - comp_geography.project_component_id, - comp_geography.feature_ids, - mpc.component_id, - comp_geography.geometry, - comp_geography.line_geometry, - comp_geography.signal_ids, - council_districts.council_districts, - 'placeholder text' AS council_districts_searchable, - NOT coalesce(council_districts IS NULL OR council_districts = '', FALSE) AS is_within_city_limits, - comp_geography.length_feet_total, - round((comp_geography.length_feet_total / 5280::numeric), 2) AS length_miles_total, - mc.component_name, - mc.component_subtype, - mc.component_name_full, - 'placeholder text' AS component_categories, - subcomponents.subcomponents AS component_subcomponents, - work_types.work_types AS component_work_types, - component_tags.component_tags, - mpc.description AS component_description, - mpc.interim_project_component_id, - mpc.completion_date, - coalesce(mpc.completion_date, substantial_completion_date) AS substantial_completion_date, - '2024-01-01T00:00:00-06:00' AS substantial_completion_date_estimated, - mpc.srts_id, - mpc.location_description AS component_location_description, - plv.project_name, - 'placeholder text' AS project_name_descriptor, - 'placeholder text' AS project_name_with_descriptor, - plv.project_description, - plv.ecapris_subproject_id, - plv.project_website, - plv.updated_at AS project_updated_at, - mpc.phase_id AS component_phase_id, - mph.phase_name AS component_phase_name, - mph.phase_name_simple AS component_phase_name_simple, - current_phase.phase_id AS project_phase_id, - current_phase.phase_name AS project_phase_name, - current_phase.phase_name_simple AS project_phase_name_simple, - coalesce(mph.phase_name, current_phase.phase_name) AS current_phase_name, - coalesce(mph.phase_name_simple, current_phase.phase_name_simple) AS current_phase_name_simple, - plv.project_team_members, - plv.project_sponsor, - plv.project_lead, - plv.public_process_status, - plv.interim_project_id, - plv.project_partners, - plv.task_order_names, - plv.funding_source_name, - plv.type_name, - plv.project_status_update, - plv.project_status_update_date_created, - plv.construction_start_date, - plv.completion_end_date, - plv.project_inspector, - plv.project_designer, - plv.project_tags, - plv.workgroup_contractors, - plv.contract_numbers, - plv.parent_project_id, - plv.parent_project_name, - plv.parent_project_url, - 'placeholder text' AS parent_project_name_with_descriptor, - 'placeholder text' AS related_project_ids, - 'placeholder text' AS related_project_ids_searchable, - plv.knack_project_id AS knack_data_tracker_project_record_id, - plv.project_url, - plv.project_url || '?tab=map&project_component_id=' || mpc.project_component_id::text AS component_url, - plv.project_development_status, - plv.project_development_status_date, - plv.project_development_status_date_calendar_year, - plv.project_development_status_date_calendar_year_month, - plv.project_development_status_date_calendar_year_month_numeric, - plv.project_development_status_date_calendar_year_quarter, - plv.project_development_status_date_fiscal_year, - plv.project_development_status_date_fiscal_year_quarter, - added_by AS project_added_by - FROM - moped_proj_components AS mpc - LEFT JOIN comp_geography ON mpc.project_component_id = comp_geography.project_component_id - LEFT JOIN council_districts ON mpc.project_component_id = council_districts.project_component_id - LEFT JOIN subcomponents ON mpc.project_component_id = subcomponents.project_component_id - LEFT JOIN work_types ON mpc.project_component_id = work_types.project_component_id - LEFT JOIN component_tags ON mpc.project_component_id = component_tags.project_component_id - LEFT JOIN project_list_view AS plv ON mpc.project_id = plv.project_id - LEFT JOIN current_phase_view AS current_phase ON mpc.project_id = current_phase.project_id - LEFT JOIN moped_phases AS mph ON mpc.phase_id = mph.phase_id - LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id - WHERE - mpc.is_deleted = FALSE - AND plv.is_deleted = FALSE -); + NOT coalesce( + ((council_districts.council_districts IS null) OR (council_districts.council_districts = ''::text)), false + ) + ) AS is_within_city_limits, + comp_geography.length_feet_total, + round(((comp_geography.length_feet_total)::numeric / (5280)::numeric), 2) AS length_miles_total, + mc.component_name, + mc.component_subtype, + mc.component_name_full, + 'placeholder text'::text AS component_categories, + subcomponents.subcomponents AS component_subcomponents, + work_types.work_types AS component_work_types, + component_tags.component_tags, + mpc.description AS component_description, + mpc.interim_project_component_id, + mpc.completion_date, + coalesce(mpc.completion_date, plv.substantial_completion_date) AS substantial_completion_date, + '2024-01-01T00:00:00-06:00'::text AS substantial_completion_date_estimated, + mpc.srts_id, + mpc.location_description AS component_location_description, + plv.project_name, + 'placeholder text'::text AS project_name_descriptor, + 'placeholder text'::text AS project_name_with_descriptor, + plv.project_description, + plv.ecapris_subproject_id, + plv.project_website, + plv.updated_at AS project_updated_at, + mpc.phase_id AS component_phase_id, + mph.phase_name AS component_phase_name, + mph.phase_name_simple AS component_phase_name_simple, + current_phase.phase_id AS project_phase_id, + current_phase.phase_name AS project_phase_name, + current_phase.phase_name_simple AS project_phase_name_simple, + coalesce(mph.phase_name, current_phase.phase_name) AS current_phase_name, + coalesce(mph.phase_name_simple, current_phase.phase_name_simple) AS current_phase_name_simple, + plv.project_team_members, + plv.project_sponsor, + plv.project_lead, + plv.public_process_status, + plv.interim_project_id, + plv.project_partners, + plv.task_order_names, + plv.funding_source_name, + plv.type_name, + plv.project_status_update, + plv.project_status_update_date_created, + plv.construction_start_date, + plv.completion_end_date, + plv.project_inspector, + plv.project_designer, + plv.project_tags, + plv.workgroup_contractors, + plv.contract_numbers, + plv.parent_project_id, + plv.parent_project_name, + plv.parent_project_url, + 'placeholder text'::text AS parent_project_name_with_descriptor, + 'placeholder text'::text AS related_project_ids, + 'placeholder text'::text AS related_project_ids_searchable, + plv.knack_project_id AS knack_data_tracker_project_record_id, + plv.project_url, + ((plv.project_url || '?tab=map&project_component_id='::text) || (mpc.project_component_id)::text) AS component_url, + plv.project_development_status, + plv.project_development_status_date, + plv.project_development_status_date_calendar_year, + plv.project_development_status_date_calendar_year_month, + plv.project_development_status_date_calendar_year_month_numeric, + plv.project_development_status_date_calendar_year_quarter, + plv.project_development_status_date_fiscal_year, + plv.project_development_status_date_fiscal_year_quarter, + plv.added_by AS project_added_by +FROM (((((((((moped_proj_components AS mpc + LEFT JOIN comp_geography ON ((mpc.project_component_id = comp_geography.project_component_id)) +) +LEFT JOIN council_districts ON ((mpc.project_component_id = council_districts.project_component_id)) +) +LEFT JOIN subcomponents ON ((mpc.project_component_id = subcomponents.project_component_id)) +) +LEFT JOIN work_types ON ((mpc.project_component_id = work_types.project_component_id)) +) +LEFT JOIN component_tags ON ((mpc.project_component_id = component_tags.project_component_id)) +) +LEFT JOIN project_list_view AS plv ON ((mpc.project_id = plv.project_id)) +) +LEFT JOIN current_phase_view AS current_phase ON ((mpc.project_id = current_phase.project_id)) +) +LEFT JOIN moped_phases AS mph ON ((mpc.phase_id = mph.phase_id)) +) +LEFT JOIN moped_components AS mc ON ((mpc.component_id = mc.component_id)) +) +WHERE ((mpc.is_deleted = false) AND (plv.is_deleted = false)); From eb9ab853fdbce2bd939f9b584b98c426b90127b0 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 7 Feb 2024 16:17:39 -0600 Subject: [PATCH 34/76] Move the blurb to the correct readme --- moped-etl/arcgis/README.md | 11 ----------- moped-etl/data-tracker-sync/README.md | 10 ++++++++++ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/moped-etl/arcgis/README.md b/moped-etl/arcgis/README.md index 46225b98d4..9b495912c3 100644 --- a/moped-etl/arcgis/README.md +++ b/moped-etl/arcgis/README.md @@ -30,14 +30,3 @@ or, to mount your local copy to a Docker container ```shell docker run -it --rm --network host --env-file env_file -v ${PWD}:/app atddocker/atd-moped-etl-arcgis:production python components_to_agol.py ``` - -### Sync evaluation script - -On first deployment of this ETL, duplicate records were created in the Knack `projects` table. The `sync_evaluation.py` script: - -- Gathers all Knack project record IDs stored in the `moped_project` table rows in the `knack_project_id` column -- Gather all Knack record IDs of all rows in the Data Tracker `projects` table -- Evaluates the overlap (synced correctly) and difference (mark for deletion) of these two lists -- Evaluates the list of differences to make sure there are no connections between Knack records marked for deletion and `work_order_signals` records -- Deletes the records with no connections and logs records with connections that were not deleted - \ No newline at end of file diff --git a/moped-etl/data-tracker-sync/README.md b/moped-etl/data-tracker-sync/README.md index 0a77e74edc..48b27ae39c 100644 --- a/moped-etl/data-tracker-sync/README.md +++ b/moped-etl/data-tracker-sync/README.md @@ -69,3 +69,13 @@ Data Tracker app and you should see that the title has been updated with your ed This script can also be run from the local Airflow stack. The secrets from the `development` sections of the **Knack AMD Data Tracker** and **Moped Hasura Admin** entries will be used automatically in the DAG. + +### Sync evaluation script + +On first deployment of this ETL, duplicate records were created in the Knack `projects` table. The `sync_evaluation.py` script: + +- Gathers all Knack project record IDs stored in the `moped_project` table rows in the `knack_project_id` column +- Gather all Knack record IDs of all rows in the Data Tracker `projects` table +- Evaluates the overlap (synced correctly) and difference (mark for deletion) of these two lists +- Evaluates the list of differences to make sure there are no connections between Knack records marked for deletion and `work_order_signals` records +- Deletes the records with no connections and logs records with connections that were not deleted From b7ce0dc40cebbd152d7262adba865b7be84d66db Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Wed, 7 Feb 2024 16:30:48 -0600 Subject: [PATCH 35/76] adds triggers and updates mutation --- moped-database/metadata/tables.yaml | 30 +++++++++---------- .../down.sql | 10 ++++--- .../up.sql | 24 ++++++++++----- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index 237df705c9..da4baa8928 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -2773,22 +2773,20 @@ action: transform template: |- { - "query": "mutation InsertActivity($object: moped_activity_log_insert_input!, $project_id:Int!, $updated_at:timestamptz ) { insert_moped_activity_log_one(object: $object) { activity_id } update_moped_project_by_pk(pk_columns: {project_id: $project_id}, _set: {updated_at: $updated_at}) { updated_at }}", - "variables": { - "object": { - "record_id": {{ $body.event.data.new.project_milestone_id }}, - "record_type": {{ $body.table.name }}, - "activity_id": {{ $body.id }}, - "record_project_id": {{ $body.event.data.new.project_id }}, - "record_data": {"event": {{ $body.event }}}, - "description": [{"newSchema": "true"}], - "operation_type": {{ $body.event.op }}, - "updated_by_user_id": {{ $session_variables?['x-hasura-user-db-id'] ?? 1}} - }, - "updated_at": {{$body.created_at}}, - "project_id": {{$body.event.data.new.project_id}} - } - } + "query": "mutation InsertActivity($object: moped_activity_log_insert_input!) { insert_moped_activity_log_one(object: $object) { activity_id } }", + "variables": { + "object": { + "record_id": {{ $body.event.data.new.project_milestone_id }}, + "record_type": {{ $body.table.name }}, + "activity_id": {{ $body.id }}, + "record_project_id": {{ $body.event.data.new.project_id }}, + "record_data": {"event": {{ $body.event }}}, + "description": [{"newSchema": "true"}], + "operation_type": {{ $body.event.op }}, + "updated_by_user_id": {{ $session_variables?['x-hasura-user-db-id'] ?? 1}} + }, + } + } method: POST query_params: {} template_engine: Kriti diff --git a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql index f6fd8ce2cc..e7bfc82feb 100644 --- a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql +++ b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/down.sql @@ -1,4 +1,6 @@ -alter table "public"."moped_proj_milestones" rename column "created_at" to "date_added"; -alter table "public"."moped_proj_milestones" drop column "created_by_user_id"; -alter table "public"."moped_proj_milestones" drop column "updated_at"; -alter table "public"."moped_proj_milestones" drop column "updated_by_user_id"; +ALTER TABLE "public"."moped_proj_milestones" RENAME COLUMN "created_at" TO "date_added"; +ALTER TABLE "public"."moped_proj_milestones" DROP COLUMN "created_by_user_id"; +ALTER TABLE "public"."moped_proj_milestones" DROP COLUMN "updated_at"; +ALTER TABLE "public"."moped_proj_milestones" DROP COLUMN "updated_by_user_id"; + +DROP TRIGGER update_moped_proj_milestones_and_project_audit_fields ON moped_proj_milestones; diff --git a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql index 65b3300bce..f25f1ae5ca 100644 --- a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql +++ b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql @@ -1,7 +1,17 @@ -alter table "public"."moped_proj_milestones" rename column "date_added" to "created_at"; -alter table "public"."moped_proj_milestones" add column "created_by_user_id" integer - null; -alter table "public"."moped_proj_milestones" add column "updated_at" timestamptz - null; -alter table "public"."moped_proj_milestones" add column "updated_by_user_id" integer - null; +ALTER TABLE "public"."moped_proj_milestones" RENAME COLUMN "date_added" TO "created_at"; +ALTER TABLE "public"."moped_proj_milestones" ADD column "created_by_user_id" INTEGER + NULL; +ALTER TABLE "public"."moped_proj_milestones" ADD column "updated_at" TIMESTAMPTZ + NULL; +ALTER TABLE "public"."moped_proj_milestones" ADD column "updated_by_user_id" INTEGER + NULL; + +COMMENT ON COLUMN moped_proj_milestones.updated_at IS 'Timestamp when the record was last updated'; +COMMENT ON COLUMN moped_proj_milestones.updated_by_user_id IS 'ID of the user who last updated the record'; + +CREATE TRIGGER update_moped_proj_milestones_and_project_audit_fields +BEFORE INSERT OR UPDATE ON moped_proj_milestones +FOR EACH ROW +EXECUTE FUNCTION public.update_self_and_project_updated_audit_fields(); + +COMMENT ON TRIGGER update_moped_proj_milestones_and_project_audit_fields ON moped_proj_funding IS 'Trigger to execute the update_self_and_project_updated_audit_fields function before each update operation on the moped_proj_milestones table.'; From 80bf31d742977285a021e1f7b42ff86877edaa66 Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:06:25 -0600 Subject: [PATCH 36/76] updates seed data --- .../up.sql | 8 ++--- .../1602292389297_initial_seed_staging.sql | 36 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql index f25f1ae5ca..4b8981e802 100644 --- a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql +++ b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql @@ -1,9 +1,9 @@ ALTER TABLE "public"."moped_proj_milestones" RENAME COLUMN "date_added" TO "created_at"; -ALTER TABLE "public"."moped_proj_milestones" ADD column "created_by_user_id" INTEGER +ALTER TABLE "public"."moped_proj_milestones" ADD COLUMN "created_by_user_id" INTEGER NULL; -ALTER TABLE "public"."moped_proj_milestones" ADD column "updated_at" TIMESTAMPTZ +ALTER TABLE "public"."moped_proj_milestones" ADD COLUMN "updated_at" TIMESTAMPTZ NULL; -ALTER TABLE "public"."moped_proj_milestones" ADD column "updated_by_user_id" INTEGER +ALTER TABLE "public"."moped_proj_milestones" ADD COLUMN "updated_by_user_id" INTEGER NULL; COMMENT ON COLUMN moped_proj_milestones.updated_at IS 'Timestamp when the record was last updated'; @@ -14,4 +14,4 @@ BEFORE INSERT OR UPDATE ON moped_proj_milestones FOR EACH ROW EXECUTE FUNCTION public.update_self_and_project_updated_audit_fields(); -COMMENT ON TRIGGER update_moped_proj_milestones_and_project_audit_fields ON moped_proj_funding IS 'Trigger to execute the update_self_and_project_updated_audit_fields function before each update operation on the moped_proj_milestones table.'; +COMMENT ON TRIGGER update_moped_proj_milestones_and_project_audit_fields ON moped_proj_milestones IS 'Trigger to execute the update_self_and_project_updated_audit_fields function before each update operation on the moped_proj_milestones table.'; diff --git a/moped-database/seeds/1602292389297_initial_seed_staging.sql b/moped-database/seeds/1602292389297_initial_seed_staging.sql index 155548749d..b9b0195417 100644 --- a/moped-database/seeds/1602292389297_initial_seed_staging.sql +++ b/moped-database/seeds/1602292389297_initial_seed_staging.sql @@ -76,24 +76,24 @@ INSERT INTO public.moped_proj_funding (proj_funding_id, project_id, created_at, -- Data for Name: moped_proj_milestones; Type: TABLE DATA; Schema: public; Owner: moped -- -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (3, 227, NULL, 2, NULL, false, '2022-11-12 18:10:41.965535+00', NULL, 35, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (4, 227, NULL, 3, NULL, false, '2022-11-12 18:10:41.965547+00', NULL, 36, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (5, 227, NULL, 4, NULL, false, '2022-11-12 18:10:41.965555+00', NULL, 37, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (6, 227, NULL, 5, NULL, false, '2022-11-12 18:10:41.965563+00', NULL, 31, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (7, 227, NULL, 6, NULL, false, '2022-11-12 18:10:41.96557+00', NULL, 38, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (8, 227, NULL, 7, NULL, false, '2022-11-12 18:10:41.965578+00', NULL, 39, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (9, 227, NULL, 8, NULL, false, '2022-11-12 18:10:41.965586+00', NULL, 40, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (10, 227, NULL, 9, NULL, false, '2022-11-12 18:10:41.965593+00', NULL, 41, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (11, 227, NULL, 10, NULL, false, '2022-11-12 18:10:41.9656+00', NULL, 42, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (12, 227, NULL, 11, NULL, false, '2022-11-12 18:10:41.965607+00', NULL, 43, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (13, 227, NULL, 12, NULL, false, '2022-11-12 18:10:41.965615+00', NULL, 44, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (14, 227, NULL, 13, NULL, false, '2022-11-12 18:10:41.965622+00', NULL, 45, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (15, 227, NULL, 14, NULL, false, '2022-11-12 18:10:41.965632+00', NULL, 46, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (16, 227, NULL, 15, NULL, false, '2022-11-12 18:10:41.96564+00', NULL, 47, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (17, 227, NULL, 16, NULL, false, '2022-11-12 18:10:41.965647+00', NULL, 48, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (18, 227, NULL, 17, NULL, false, '2022-11-12 18:10:41.965654+00', NULL, 49, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (19, 227, NULL, 18, NULL, false, '2022-11-12 18:10:41.965664+00', NULL, 50, false); -INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, date_added, date_estimate, milestone_id, is_deleted) VALUES (20, 227, NULL, 19, NULL, false, '2022-11-12 18:10:41.965672+00', NULL, 51, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (3, 227, NULL, 2, NULL, false, '2022-11-12 18:10:41.965535+00', NULL, 35, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (4, 227, NULL, 3, NULL, false, '2022-11-12 18:10:41.965547+00', NULL, 36, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (5, 227, NULL, 4, NULL, false, '2022-11-12 18:10:41.965555+00', NULL, 37, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (6, 227, NULL, 5, NULL, false, '2022-11-12 18:10:41.965563+00', NULL, 31, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (7, 227, NULL, 6, NULL, false, '2022-11-12 18:10:41.96557+00', NULL, 38, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (8, 227, NULL, 7, NULL, false, '2022-11-12 18:10:41.965578+00', NULL, 39, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (9, 227, NULL, 8, NULL, false, '2022-11-12 18:10:41.965586+00', NULL, 40, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (10, 227, NULL, 9, NULL, false, '2022-11-12 18:10:41.965593+00', NULL, 41, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (11, 227, NULL, 10, NULL, false, '2022-11-12 18:10:41.9656+00', NULL, 42, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (12, 227, NULL, 11, NULL, false, '2022-11-12 18:10:41.965607+00', NULL, 43, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (13, 227, NULL, 12, NULL, false, '2022-11-12 18:10:41.965615+00', NULL, 44, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (14, 227, NULL, 13, NULL, false, '2022-11-12 18:10:41.965622+00', NULL, 45, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (15, 227, NULL, 14, NULL, false, '2022-11-12 18:10:41.965632+00', NULL, 46, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (16, 227, NULL, 15, NULL, false, '2022-11-12 18:10:41.96564+00', NULL, 47, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (17, 227, NULL, 16, NULL, false, '2022-11-12 18:10:41.965647+00', NULL, 48, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (18, 227, NULL, 17, NULL, false, '2022-11-12 18:10:41.965654+00', NULL, 49, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (19, 227, NULL, 18, NULL, false, '2022-11-12 18:10:41.965664+00', NULL, 50, false); +INSERT INTO public.moped_proj_milestones (project_milestone_id, project_id, description, milestone_order, date_actual, completed, created_at, date_estimate, milestone_id, is_deleted) VALUES (20, 227, NULL, 19, NULL, false, '2022-11-12 18:10:41.965672+00', NULL, 51, false); -- From 8d0edc65ff94780087f0fdb3248a06f11642fd2e Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:49:47 -0600 Subject: [PATCH 37/76] this should do it --- .../activityLogFormatters/mopedMilestonesActivity.js | 6 +++++- .../projects/projectView/ProjectActivityLogTableMaps.js | 9 ++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/moped-editor/src/utils/activityLogFormatters/mopedMilestonesActivity.js b/moped-editor/src/utils/activityLogFormatters/mopedMilestonesActivity.js index 62415100d8..b6d7a7fa36 100644 --- a/moped-editor/src/utils/activityLogFormatters/mopedMilestonesActivity.js +++ b/moped-editor/src/utils/activityLogFormatters/mopedMilestonesActivity.js @@ -45,7 +45,11 @@ export const formatMilestonesActivity = (change, milestoneList) => { // loop through fields to check for differences, push label onto changes Array Object.keys(newRecord).forEach((field) => { if (newRecord[field] !== oldRecord[field]) { - changes.push(entryMap.fields[field]?.label); + // filter out fields that are not listed in the activity log table maps to prevent + // automated field updates (created at, updated at, etc.) from entering the array + if (!!entryMap.fields[field]) { + changes.push(entryMap.fields[field]?.label); + } } }); diff --git a/moped-editor/src/views/projects/projectView/ProjectActivityLogTableMaps.js b/moped-editor/src/views/projects/projectView/ProjectActivityLogTableMaps.js index ddba572942..0928ba9a33 100644 --- a/moped-editor/src/views/projects/projectView/ProjectActivityLogTableMaps.js +++ b/moped-editor/src/views/projects/projectView/ProjectActivityLogTableMaps.js @@ -151,9 +151,6 @@ export const ProjectActivityLogTableMaps = { milestone_name: { label: "name", }, - date_added: { - label: "date added", - }, milestone_estimate: { label: "completion estimate", }, @@ -163,6 +160,12 @@ export const ProjectActivityLogTableMaps = { is_deleted: { label: "is deleted", }, + created_at: { + label: "date added", + }, + created_by_user_id: { + label: "added by", + }, }, }, moped_proj_notes: { From ba892a1fd255dc43e4a48925f8137ef740108a2b Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 7 Feb 2024 18:19:35 -0600 Subject: [PATCH 38/76] Determine which query to build based on present search term and adv search filters --- .../useProjectListViewQuery.js | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js index be0b7c59de..11a2c5f9cd 100644 --- a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js +++ b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js @@ -12,22 +12,50 @@ export const useGetProjectListView = ({ advancedSearchWhereString, }) => { const { query, exportQuery } = useMemo(() => { + console.log({ searchWhereString, advancedSearchWhereString }); + const queryString = `query ProjectListView { project_list_view ( limit: ${queryLimit} offset: ${queryOffset} order_by: {${orderByColumn}: ${orderByDirection}} where: { - ${advancedSearchWhereString ? advancedSearchWhereString : ""} - ${searchWhereString ? `_or: [${searchWhereString}]` : ""} + ${ + searchWhereString && advancedSearchWhereString + ? `_or: [${searchWhereString}, {${advancedSearchWhereString}}]` + : "" + } + ${ + searchWhereString && !advancedSearchWhereString + ? `_or: [${searchWhereString}]` + : "" + } + ${ + advancedSearchWhereString && !searchWhereString + ? advancedSearchWhereString + : "" + } } ) { ${columnsToReturn.join("\n")} }, project_list_view_aggregate ( where: { - ${advancedSearchWhereString ? advancedSearchWhereString : ""} - ${searchWhereString ? `_or: [${searchWhereString}]` : ""} + ${ + searchWhereString && advancedSearchWhereString + ? `_or: [${searchWhereString}, {${advancedSearchWhereString}}]` + : "" + } + ${ + searchWhereString && !advancedSearchWhereString + ? `_or: [${searchWhereString}]` + : "" + } + ${ + advancedSearchWhereString && !searchWhereString + ? advancedSearchWhereString + : "" + } } ) { aggregate { @@ -35,6 +63,7 @@ export const useGetProjectListView = ({ } } }`; + console.log(queryString); const query = gql` ${queryString} `; From c7b99ac4a96c3fb7fed870eba390bad01e245634 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 7 Feb 2024 18:19:58 -0600 Subject: [PATCH 39/76] Use search term and advanced filters instead of or --- .../useProjectListViewQuery/useProjectListViewQuery.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js index 11a2c5f9cd..dc48c1a5b1 100644 --- a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js +++ b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js @@ -22,7 +22,7 @@ export const useGetProjectListView = ({ where: { ${ searchWhereString && advancedSearchWhereString - ? `_or: [${searchWhereString}, {${advancedSearchWhereString}}]` + ? `_and: [{_or: [${searchWhereString}]}, {${advancedSearchWhereString}}]` : "" } ${ @@ -43,7 +43,7 @@ export const useGetProjectListView = ({ where: { ${ searchWhereString && advancedSearchWhereString - ? `_or: [${searchWhereString}, {${advancedSearchWhereString}}]` + ? `_and: [{_or: [${searchWhereString}]}, {${advancedSearchWhereString}}]` : "" } ${ From fd82fd6de3d5ff93ce79ba98ea5f28865b7000c6 Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 8 Feb 2024 11:45:58 -0600 Subject: [PATCH 40/76] Remove logs --- .../useProjectListViewQuery/useProjectListViewQuery.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js index dc48c1a5b1..b0e4cf043c 100644 --- a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js +++ b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useProjectListViewQuery.js @@ -12,8 +12,6 @@ export const useGetProjectListView = ({ advancedSearchWhereString, }) => { const { query, exportQuery } = useMemo(() => { - console.log({ searchWhereString, advancedSearchWhereString }); - const queryString = `query ProjectListView { project_list_view ( limit: ${queryLimit} @@ -63,7 +61,6 @@ export const useGetProjectListView = ({ } } }`; - console.log(queryString); const query = gql` ${queryString} `; From 9d934e65ee8378804447ad99ea6f5f652612797a Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 8 Feb 2024 12:10:20 -0600 Subject: [PATCH 41/76] create CTE via unwrap/array_agg that contains a project and it's distinct set of council districts that its components lie in --- moped-database/views/project_list_view.sql | 27 ++++++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/moped-database/views/project_list_view.sql b/moped-database/views/project_list_view.sql index 1e73691c4a..7457331c20 100644 --- a/moped-database/views/project_list_view.sql +++ b/moped-database/views/project_list_view.sql @@ -1,6 +1,5 @@ --- latest version 1706897389736_fix_sub_comp_date_dash -CREATE OR REPLACE VIEW public.project_list_view -AS WITH project_person_list_lookup AS ( +CREATE OR REPLACE VIEW public.project_list_view AS +WITH project_person_list_lookup AS ( SELECT mpp.project_id, string_agg( @@ -82,6 +81,21 @@ moped_proj_components_subtypes AS ( LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id WHERE mpc.is_deleted = false GROUP BY mpc.project_id +), + +project_district_association AS ( + SELECT + flattened_project_council_map.project_id, + array_agg(DISTINCT flattened_project_council_map.single_districts) AS council_districts + FROM ( + SELECT + components.project_id, + unnest(features.council_districts) AS single_districts + FROM moped_proj_components AS components + INNER JOIN uniform_features AS features ON (components.component_id = features.component_id) + WHERE components.is_deleted IS false + ) AS flattened_project_council_map + GROUP BY flattened_project_council_map.project_id ) SELECT @@ -236,7 +250,8 @@ SELECT GROUP BY ptags.project_id ) AS project_tags, concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, - mpcs.components + mpcs.components, + districts.council_districts FROM moped_project AS mp LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id @@ -251,6 +266,7 @@ LEFT JOIN current_phase_view AS current_phase ON mp.project_id = current_phase.p LEFT JOIN moped_public_process_statuses AS mpps ON mp.public_process_status_id = mpps.id LEFT JOIN child_project_lookup AS cpl ON mp.project_id = cpl.parent_id LEFT JOIN moped_proj_components_subtypes AS mpcs ON mp.project_id = mpcs.project_id +LEFT JOIN project_district_association AS districts ON (mp.project_id = districts.project_id) LEFT JOIN LATERAL ( SELECT @@ -293,4 +309,5 @@ GROUP BY work_activities.contract_numbers, work_activities.task_order_names, work_activities.task_order_names_short, - work_activities.task_orders; + work_activities.task_orders, + districts.council_districts; From ee22c7a1d28294fc97a7dcbcc124f885efa6d08c Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 8 Feb 2024 12:19:28 -0600 Subject: [PATCH 42/76] Add actual migration --- .../down.sql | 298 +++++++++++++++++ .../up.sql | 316 ++++++++++++++++++ 2 files changed, 614 insertions(+) create mode 100644 moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql create mode 100644 moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql new file mode 100644 index 0000000000..9d5d57a23e --- /dev/null +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql @@ -0,0 +1,298 @@ +-- no "IF EXISTS" because if it doesn't exist, we have bigger problems and we want this to bail +DROP VIEW public.project_list_view; + +CREATE OR REPLACE VIEW public.project_list_view +AS WITH project_person_list_lookup AS ( + SELECT + mpp.project_id, + string_agg( + DISTINCT concat(mu.first_name, ' ', mu.last_name, ':', mpr.project_role_name), ','::text + ) AS project_team_members + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS mu ON mpp.user_id = mu.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE mpp.is_deleted = false + AND mppr.is_deleted = false + GROUP BY mpp.project_id +), + +funding_sources_lookup AS ( + SELECT + mpf_1.project_id, + string_agg(mfs.funding_source_name, ', '::text) AS funding_source_name + FROM moped_proj_funding AS mpf_1 + LEFT JOIN moped_fund_sources AS mfs ON mpf_1.funding_source_id = mfs.funding_source_id + WHERE mpf_1.is_deleted = false + GROUP BY mpf_1.project_id +), + +project_type_lookup AS ( + SELECT + mpt.project_id, + string_agg(mt.type_name, ', '::text) AS type_name + FROM moped_project_types AS mpt + LEFT JOIN moped_types AS mt ON mpt.project_type_id = mt.type_id AND mpt.is_deleted = false + GROUP BY mpt.project_id +), + +child_project_lookup AS ( + SELECT + jsonb_agg(children.project_id) AS children_project_ids, + children.parent_project_id AS parent_id + FROM moped_project AS children + INNER JOIN moped_project AS parent ON (children.parent_project_id = parent.project_id) + WHERE children.is_deleted = false + GROUP BY parent_id +), + +work_activities AS ( + SELECT + project_id, + string_agg( + task_order_objects.task_order_object ->> 'display_name'::text, + ', '::text + ) AS task_order_names, + string_agg( + task_order_objects.task_order_object ->> 'task_order'::text, + ', '::text + ) AS task_order_names_short, + jsonb_agg(DISTINCT task_order_objects.task_order_object) FILTER ( + WHERE task_order_objects.task_order_object IS NOT null + ) AS task_orders, + string_agg( + DISTINCT mpwa.workgroup_contractor, + ', '::text + ) AS workgroup_contractors, + string_agg( + mpwa.contract_number, + ', '::text + ) AS contract_numbers + FROM moped_proj_work_activity AS mpwa + LEFT JOIN LATERAL jsonb_array_elements(mpwa.task_orders) AS task_order_objects (task_order_object) + ON true WHERE 1 = 1 + AND mpwa.is_deleted = false + GROUP BY + mpwa.project_id +), + +moped_proj_components_subtypes AS ( + SELECT + mpc.project_id, + string_agg(DISTINCT mc.component_name_full, ', '::text) AS components + FROM moped_proj_components AS mpc + LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id + WHERE mpc.is_deleted = false + GROUP BY mpc.project_id +) + +SELECT + mp.project_id, + mp.project_name, + mp.project_description, + mp.ecapris_subproject_id, + mp.project_website, + mp.date_added, + mp.is_deleted, + mp.updated_at, + current_phase.phase_name AS current_phase, + current_phase.phase_key AS current_phase_key, + current_phase.phase_name_simple AS current_phase_simple, + ppll.project_team_members, + me.entity_name AS project_sponsor, + mel.entity_name AS project_lead, + mpps.name AS public_process_status, + mp.interim_project_id, + mp.parent_project_id, + mp.knack_project_id, + 'https://mobility.austin.gov/moped/projects/' || mp.project_id::text AS project_url, + 'https://mobility.austin.gov/moped/projects/' || mp.parent_project_id::text AS parent_project_url, + proj_status_update.project_note AS project_status_update, + proj_status_update.date_created AS project_status_update_date_created, + work_activities.workgroup_contractors, + work_activities.contract_numbers, + work_activities.task_order_names, + work_activities.task_order_names_short, + work_activities.task_orders, + 'placeholder text' AS project_development_status, + '2024-01-01T00:00:00-06:00' AS project_development_status_date, + 9999 AS project_development_status_date_calendar_year, + 'placeholder text' AS project_development_status_date_calendar_year_month, + 'placeholder text' AS project_development_status_date_calendar_year_month_numeric, + 'placeholder text' AS project_development_status_date_calendar_year_quarter, + 999 AS project_development_status_date_fiscal_year, + 'placeholder text' AS project_development_status_date_fiscal_year_quarter, + ( + SELECT project_name + FROM moped_project + WHERE project_id = mp.parent_project_id + ) AS parent_project_name, + cpl.children_project_ids, + string_agg(DISTINCT me2.entity_name, ', '::text) AS project_partners, + ( + SELECT + json_agg( + json_build_object( + 'signal_id', + feature_signals.signal_id, + 'knack_id', + feature_signals.knack_id, + 'location_name', + feature_signals.location_name, + 'signal_type', + feature_signals.signal_type, + 'id', + feature_signals.id + ) + ) + FROM moped_proj_components AS components + LEFT JOIN feature_signals + ON (components.project_component_id = feature_signals.component_id) + WHERE true + AND components.is_deleted = false + AND components.project_id = mp.project_id + AND feature_signals.signal_id IS NOT null + AND feature_signals.is_deleted = false + ) AS project_feature, + fsl.funding_source_name, + ptl.type_name, + ( -- get the date of the construction phase with the earliest start date + SELECT min(phases.phase_start) + FROM moped_proj_phases AS phases + WHERE true + AND phases.project_id = mp.project_id + AND phases.phase_id = 9 -- phase_id 9 is construction + AND phases.is_deleted = false + ) AS construction_start_date, + ( -- get the date of the completion phase with the latest end date + SELECT max(phases.phase_end) + FROM moped_proj_phases AS phases + WHERE true + AND phases.project_id = mp.project_id + AND phases.phase_id = 11 -- phase_id 11 is complete + AND phases.is_deleted = false + ) AS completion_end_date, + ( -- get the earliest confirmed phase_start or phase_end with a simple phase of 'Complete' + SELECT min(min_confirmed_date) + FROM ( + -- earliest confirmed phase start + SELECT min(phases.phase_start) AS min_confirmed_date + FROM + moped_proj_phases AS phases + LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id + WHERE + true + AND phases.phase_start IS NOT null + AND phases.is_phase_start_confirmed = true + AND phases.project_id = mp.project_id + AND moped_phases.phase_name_simple = 'Complete' + AND phases.is_deleted = false + UNION ALL + -- earliest confirmed phase end + SELECT min(phases.phase_end) AS min_confirmed_date + FROM + moped_proj_phases AS phases + LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id + WHERE + true + AND phases.phase_end IS NOT null + AND phases.is_phase_end_confirmed = true + AND phases.project_id = mp.project_id + AND moped_phases.phase_name_simple = 'Complete' + AND phases.is_deleted = false + ) AS min_confirmed_dates + ) AS substantial_completion_date, + ( -- get me a list of the inspectors for this project + SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS users ON mpp.user_id = users.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE 1 = 1 + AND mpr.project_role_name = 'Inspector'::text + AND mpp.is_deleted = false + AND mppr.is_deleted = false + AND mpp.project_id = mp.project_id + GROUP BY mpp.project_id + ) AS project_inspector, + ( -- get me a list of the designers for this project + SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS users ON mpp.user_id = users.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE 1 = 1 + AND mpr.project_role_name = 'Designer'::text + AND mpp.is_deleted = false + AND mppr.is_deleted = false + AND mpp.project_id = mp.project_id + GROUP BY mpp.project_id + ) AS project_designer, + ( -- get me all of the tags added to a project + SELECT string_agg(tags.name, ', '::text) AS string_agg + FROM moped_proj_tags AS ptags + INNER JOIN moped_tags AS tags ON ptags.tag_id = tags.id + WHERE 1 = 1 + AND ptags.is_deleted = false + AND ptags.project_id = mp.project_id + GROUP BY ptags.project_id + ) AS project_tags, + concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, + mpcs.components +FROM moped_project AS mp +LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id +LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id +LEFT JOIN project_type_lookup AS ptl ON mp.project_id = ptl.project_id +LEFT JOIN moped_entity AS me ON mp.project_sponsor = me.entity_id +LEFT JOIN moped_entity AS mel ON mp.project_lead_id = mel.entity_id +LEFT JOIN moped_proj_partners AS mpp2 ON mp.project_id = mpp2.project_id AND mpp2.is_deleted = false +LEFT JOIN moped_entity AS me2 ON mpp2.entity_id = me2.entity_id +LEFT JOIN work_activities ON mp.project_id = work_activities.project_id +LEFT JOIN moped_users AS added_by_user ON mp.added_by = added_by_user.user_id +LEFT JOIN current_phase_view AS current_phase ON mp.project_id = current_phase.project_id +LEFT JOIN moped_public_process_statuses AS mpps ON mp.public_process_status_id = mpps.id +LEFT JOIN child_project_lookup AS cpl ON mp.project_id = cpl.parent_id +LEFT JOIN moped_proj_components_subtypes AS mpcs ON mp.project_id = mpcs.project_id +LEFT JOIN LATERAL + ( + SELECT + mpn.project_note, + mpn.date_created + FROM moped_proj_notes AS mpn + WHERE mpn.project_id = mp.project_id AND mpn.project_note_type = 2 AND mpn.is_deleted = false + ORDER BY mpn.date_created DESC + LIMIT 1 + ) AS proj_status_update ON true +WHERE + mp.is_deleted = false +GROUP BY + mp.project_id, + mp.project_name, + mp.project_description, + ppll.project_team_members, + mp.ecapris_subproject_id, + mp.date_added, + mp.is_deleted, + me.entity_name, + mel.entity_name, + mp.updated_at, + mp.interim_project_id, + mp.parent_project_id, + mp.knack_project_id, + current_phase.phase_name, + current_phase.phase_key, + current_phase.phase_name_simple, + ptl.type_name, + mpcs.components, + fsl.funding_source_name, + added_by_user.first_name, + added_by_user.last_name, + mpps.name, + cpl.children_project_ids, + proj_status_update.project_note, + proj_status_update.date_created, + work_activities.workgroup_contractors, + work_activities.contract_numbers, + work_activities.task_order_names, + work_activities.task_order_names_short, + work_activities.task_orders; \ No newline at end of file diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql new file mode 100644 index 0000000000..2bbb2b1eba --- /dev/null +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql @@ -0,0 +1,316 @@ +-- no "IF EXISTS" because if it doesn't exist, we have bigger problems and we want this to bail +DROP VIEW public.project_list_view; + +CREATE OR REPLACE VIEW public.project_list_view AS +WITH project_person_list_lookup AS ( + SELECT + mpp.project_id, + string_agg( + DISTINCT concat(mu.first_name, ' ', mu.last_name, ':', mpr.project_role_name), ','::text + ) AS project_team_members + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS mu ON mpp.user_id = mu.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE mpp.is_deleted = false + AND mppr.is_deleted = false + GROUP BY mpp.project_id +), + +funding_sources_lookup AS ( + SELECT + mpf_1.project_id, + string_agg(mfs.funding_source_name, ', '::text) AS funding_source_name + FROM moped_proj_funding AS mpf_1 + LEFT JOIN moped_fund_sources AS mfs ON mpf_1.funding_source_id = mfs.funding_source_id + WHERE mpf_1.is_deleted = false + GROUP BY mpf_1.project_id +), + +project_type_lookup AS ( + SELECT + mpt.project_id, + string_agg(mt.type_name, ', '::text) AS type_name + FROM moped_project_types AS mpt + LEFT JOIN moped_types AS mt ON mpt.project_type_id = mt.type_id AND mpt.is_deleted = false + GROUP BY mpt.project_id +), + +child_project_lookup AS ( + SELECT + jsonb_agg(children.project_id) AS children_project_ids, + children.parent_project_id AS parent_id + FROM moped_project AS children + INNER JOIN moped_project AS parent ON (children.parent_project_id = parent.project_id) + WHERE children.is_deleted = false + GROUP BY parent_id +), + +work_activities AS ( + SELECT + project_id, + string_agg( + task_order_objects.task_order_object ->> 'display_name'::text, + ', '::text + ) AS task_order_names, + string_agg( + task_order_objects.task_order_object ->> 'task_order'::text, + ', '::text + ) AS task_order_names_short, + jsonb_agg(DISTINCT task_order_objects.task_order_object) FILTER ( + WHERE task_order_objects.task_order_object IS NOT null + ) AS task_orders, + string_agg( + DISTINCT mpwa.workgroup_contractor, + ', '::text + ) AS workgroup_contractors, + string_agg( + mpwa.contract_number, + ', '::text + ) AS contract_numbers + FROM moped_proj_work_activity AS mpwa + LEFT JOIN LATERAL jsonb_array_elements(mpwa.task_orders) AS task_order_objects (task_order_object) + ON true WHERE 1 = 1 + AND mpwa.is_deleted = false + GROUP BY + mpwa.project_id +), + +moped_proj_components_subtypes AS ( + SELECT + mpc.project_id, + string_agg(DISTINCT mc.component_name_full, ', '::text) AS components + FROM moped_proj_components AS mpc + LEFT JOIN moped_components AS mc ON mpc.component_id = mc.component_id + WHERE mpc.is_deleted = false + GROUP BY mpc.project_id +), + +project_district_association AS ( + SELECT + flattened_project_council_map.project_id, + array_agg(DISTINCT flattened_project_council_map.single_districts) AS council_districts + FROM ( + SELECT + components.project_id, + unnest(features.council_districts) AS single_districts + FROM moped_proj_components AS components + INNER JOIN uniform_features AS features ON (components.component_id = features.component_id) + WHERE components.is_deleted IS false + ) AS flattened_project_council_map + GROUP BY flattened_project_council_map.project_id +) + +SELECT + mp.project_id, + mp.project_name, + mp.project_description, + mp.ecapris_subproject_id, + mp.project_website, + mp.date_added, + mp.is_deleted, + mp.updated_at, + current_phase.phase_name AS current_phase, + current_phase.phase_key AS current_phase_key, + current_phase.phase_name_simple AS current_phase_simple, + ppll.project_team_members, + me.entity_name AS project_sponsor, + mel.entity_name AS project_lead, + mpps.name AS public_process_status, + mp.interim_project_id, + mp.parent_project_id, + mp.knack_project_id, + 'https://mobility.austin.gov/moped/projects/' || mp.project_id::text AS project_url, + 'https://mobility.austin.gov/moped/projects/' || mp.parent_project_id::text AS parent_project_url, + proj_status_update.project_note AS project_status_update, + proj_status_update.date_created AS project_status_update_date_created, + work_activities.workgroup_contractors, + work_activities.contract_numbers, + work_activities.task_order_names, + work_activities.task_order_names_short, + work_activities.task_orders, + 'placeholder text' AS project_development_status, + '2024-01-01T00:00:00-06:00' AS project_development_status_date, + 9999 AS project_development_status_date_calendar_year, + 'placeholder text' AS project_development_status_date_calendar_year_month, + 'placeholder text' AS project_development_status_date_calendar_year_month_numeric, + 'placeholder text' AS project_development_status_date_calendar_year_quarter, + 999 AS project_development_status_date_fiscal_year, + 'placeholder text' AS project_development_status_date_fiscal_year_quarter, + ( + SELECT project_name + FROM moped_project + WHERE project_id = mp.parent_project_id + ) AS parent_project_name, + cpl.children_project_ids, + string_agg(DISTINCT me2.entity_name, ', '::text) AS project_partners, + ( + SELECT + json_agg( + json_build_object( + 'signal_id', + feature_signals.signal_id, + 'knack_id', + feature_signals.knack_id, + 'location_name', + feature_signals.location_name, + 'signal_type', + feature_signals.signal_type, + 'id', + feature_signals.id + ) + ) + FROM moped_proj_components AS components + LEFT JOIN feature_signals + ON (components.project_component_id = feature_signals.component_id) + WHERE true + AND components.is_deleted = false + AND components.project_id = mp.project_id + AND feature_signals.signal_id IS NOT null + AND feature_signals.is_deleted = false + ) AS project_feature, + fsl.funding_source_name, + ptl.type_name, + ( -- get the date of the construction phase with the earliest start date + SELECT min(phases.phase_start) + FROM moped_proj_phases AS phases + WHERE true + AND phases.project_id = mp.project_id + AND phases.phase_id = 9 -- phase_id 9 is construction + AND phases.is_deleted = false + ) AS construction_start_date, + ( -- get the date of the completion phase with the latest end date + SELECT max(phases.phase_end) + FROM moped_proj_phases AS phases + WHERE true + AND phases.project_id = mp.project_id + AND phases.phase_id = 11 -- phase_id 11 is complete + AND phases.is_deleted = false + ) AS completion_end_date, + ( -- get the earliest confirmed phase_start or phase_end with a simple phase of 'Complete' + SELECT min(min_confirmed_date) + FROM ( + -- earliest confirmed phase start + SELECT min(phases.phase_start) AS min_confirmed_date + FROM + moped_proj_phases AS phases + LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id + WHERE + true + AND phases.phase_start IS NOT null + AND phases.is_phase_start_confirmed = true + AND phases.project_id = mp.project_id + AND moped_phases.phase_name_simple = 'Complete' + AND phases.is_deleted = false + UNION ALL + -- earliest confirmed phase end + SELECT min(phases.phase_end) AS min_confirmed_date + FROM + moped_proj_phases AS phases + LEFT JOIN moped_phases ON phases.phase_id = moped_phases.phase_id + WHERE + true + AND phases.phase_end IS NOT null + AND phases.is_phase_end_confirmed = true + AND phases.project_id = mp.project_id + AND moped_phases.phase_name_simple = 'Complete' + AND phases.is_deleted = false + ) AS min_confirmed_dates + ) AS substantial_completion_date, + ( -- get me a list of the inspectors for this project + SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS users ON mpp.user_id = users.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE 1 = 1 + AND mpr.project_role_name = 'Inspector'::text + AND mpp.is_deleted = false + AND mppr.is_deleted = false + AND mpp.project_id = mp.project_id + GROUP BY mpp.project_id + ) AS project_inspector, + ( -- get me a list of the designers for this project + SELECT string_agg(concat(users.first_name, ' ', users.last_name), ', '::text) AS string_agg + FROM moped_proj_personnel AS mpp + INNER JOIN moped_users AS users ON mpp.user_id = users.user_id + INNER JOIN moped_proj_personnel_roles AS mppr ON mpp.project_personnel_id = mppr.project_personnel_id + INNER JOIN moped_project_roles AS mpr ON mppr.project_role_id = mpr.project_role_id + WHERE 1 = 1 + AND mpr.project_role_name = 'Designer'::text + AND mpp.is_deleted = false + AND mppr.is_deleted = false + AND mpp.project_id = mp.project_id + GROUP BY mpp.project_id + ) AS project_designer, + ( -- get me all of the tags added to a project + SELECT string_agg(tags.name, ', '::text) AS string_agg + FROM moped_proj_tags AS ptags + INNER JOIN moped_tags AS tags ON ptags.tag_id = tags.id + WHERE 1 = 1 + AND ptags.is_deleted = false + AND ptags.project_id = mp.project_id + GROUP BY ptags.project_id + ) AS project_tags, + concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, + mpcs.components, + districts.council_districts +FROM moped_project AS mp +LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id +LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id +LEFT JOIN project_type_lookup AS ptl ON mp.project_id = ptl.project_id +LEFT JOIN moped_entity AS me ON mp.project_sponsor = me.entity_id +LEFT JOIN moped_entity AS mel ON mp.project_lead_id = mel.entity_id +LEFT JOIN moped_proj_partners AS mpp2 ON mp.project_id = mpp2.project_id AND mpp2.is_deleted = false +LEFT JOIN moped_entity AS me2 ON mpp2.entity_id = me2.entity_id +LEFT JOIN work_activities ON mp.project_id = work_activities.project_id +LEFT JOIN moped_users AS added_by_user ON mp.added_by = added_by_user.user_id +LEFT JOIN current_phase_view AS current_phase ON mp.project_id = current_phase.project_id +LEFT JOIN moped_public_process_statuses AS mpps ON mp.public_process_status_id = mpps.id +LEFT JOIN child_project_lookup AS cpl ON mp.project_id = cpl.parent_id +LEFT JOIN moped_proj_components_subtypes AS mpcs ON mp.project_id = mpcs.project_id +LEFT JOIN project_district_association AS districts ON (mp.project_id = districts.project_id) +LEFT JOIN LATERAL + ( + SELECT + mpn.project_note, + mpn.date_created + FROM moped_proj_notes AS mpn + WHERE mpn.project_id = mp.project_id AND mpn.project_note_type = 2 AND mpn.is_deleted = false + ORDER BY mpn.date_created DESC + LIMIT 1 + ) AS proj_status_update ON true +WHERE + mp.is_deleted = false +GROUP BY + mp.project_id, + mp.project_name, + mp.project_description, + ppll.project_team_members, + mp.ecapris_subproject_id, + mp.date_added, + mp.is_deleted, + me.entity_name, + mel.entity_name, + mp.updated_at, + mp.interim_project_id, + mp.parent_project_id, + mp.knack_project_id, + current_phase.phase_name, + current_phase.phase_key, + current_phase.phase_name_simple, + ptl.type_name, + mpcs.components, + fsl.funding_source_name, + added_by_user.first_name, + added_by_user.last_name, + mpps.name, + cpl.children_project_ids, + proj_status_update.project_note, + proj_status_update.date_created, + work_activities.workgroup_contractors, + work_activities.contract_numbers, + work_activities.task_order_names, + work_activities.task_order_names_short, + work_activities.task_orders, + districts.council_districts; From b21f71fa4df7bce22658b3e093e766cc5e075efb Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 8 Feb 2024 12:22:07 -0600 Subject: [PATCH 43/76] Gotta drop this and it's logical children too --- .../down.sql | 2 +- .../1707416196011_add_council_districts_to_project_view/up.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql index 9d5d57a23e..130870a341 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql @@ -1,5 +1,5 @@ -- no "IF EXISTS" because if it doesn't exist, we have bigger problems and we want this to bail -DROP VIEW public.project_list_view; +DROP VIEW public.project_list_view CASCADE; CREATE OR REPLACE VIEW public.project_list_view AS WITH project_person_list_lookup AS ( diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql index 2bbb2b1eba..ad88137e1e 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql @@ -1,5 +1,5 @@ -- no "IF EXISTS" because if it doesn't exist, we have bigger problems and we want this to bail -DROP VIEW public.project_list_view; +DROP VIEW public.project_list_view CASCADE; CREATE OR REPLACE VIEW public.project_list_view AS WITH project_person_list_lookup AS ( From 5063842f5802785d0fbca8e77ca6c3c35bb0ef85 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 8 Feb 2024 12:27:56 -0600 Subject: [PATCH 44/76] pull in the GIS view as a dependent relation --- .../down.sql | 223 ++++++++++++++++- .../up.sql | 224 ++++++++++++++++++ 2 files changed, 446 insertions(+), 1 deletion(-) diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql index 130870a341..ed624bab2f 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql @@ -295,4 +295,225 @@ GROUP BY work_activities.contract_numbers, work_activities.task_order_names, work_activities.task_order_names_short, - work_activities.task_orders; \ No newline at end of file + work_activities.task_orders; + + + + +CREATE VIEW component_arcgis_online_view AS +WITH work_types AS ( + SELECT + mpcwt.project_component_id, + string_agg(mwt.name, ', '::text) AS work_types + FROM ( + moped_proj_component_work_types AS mpcwt + LEFT JOIN moped_work_types AS mwt ON ((mpcwt.work_type_id = mwt.id)) + ) + WHERE (mpcwt.is_deleted = false) + GROUP BY mpcwt.project_component_id +), + +council_districts AS ( + SELECT + features.component_id AS project_component_id, + string_agg(DISTINCT (features_council_districts.council_district_id)::text, ', '::text) AS council_districts + FROM ( + features_council_districts + LEFT JOIN features ON ((features_council_districts.feature_id = features.id)) + ) + WHERE (features.is_deleted = false) + GROUP BY features.component_id +), + +comp_geography AS ( + SELECT + feature_union.component_id AS project_component_id, + string_agg(DISTINCT (feature_union.id)::text, ', '::text) AS feature_ids, + (st_asgeojson(st_union(array_agg(feature_union.geography))))::json AS geometry, + (st_asgeojson(st_union(array_agg(feature_union.line_geography))))::json AS line_geometry, + string_agg(DISTINCT (feature_union.signal_id)::text, ', '::text) AS signal_ids, + sum(feature_union.length_feet) AS length_feet_total + FROM ( + SELECT + feature_signals.id, + feature_signals.component_id, + (feature_signals.geography)::geometry AS geography, + st_exteriorring((st_buffer(feature_signals.geography, (7)::double precision))::geometry) AS line_geography, + feature_signals.signal_id, + null::integer AS length_feet + FROM feature_signals + WHERE (feature_signals.is_deleted = false) + UNION ALL + SELECT + feature_street_segments.id, + feature_street_segments.component_id, + (feature_street_segments.geography)::geometry AS geography, + (feature_street_segments.geography)::geometry AS line_geography, + null::integer AS signal_id, + feature_street_segments.length_feet + FROM feature_street_segments + WHERE (feature_street_segments.is_deleted = false) + UNION ALL + SELECT + feature_intersections.id, + feature_intersections.component_id, + (feature_intersections.geography)::geometry AS geography, + st_exteriorring( + (st_buffer(feature_intersections.geography, (7)::double precision))::geometry + ) AS line_geography, + null::integer AS signal_id, + null::integer AS length_feet + FROM feature_intersections + WHERE (feature_intersections.is_deleted = false) + UNION ALL + SELECT + feature_drawn_points.id, + feature_drawn_points.component_id, + (feature_drawn_points.geography)::geometry AS geography, + st_exteriorring( + (st_buffer(feature_drawn_points.geography, (7)::double precision))::geometry + ) AS line_geography, + null::integer AS signal_id, + null::integer AS length_feet + FROM feature_drawn_points + WHERE (feature_drawn_points.is_deleted = false) + UNION ALL + SELECT + feature_drawn_lines.id, + feature_drawn_lines.component_id, + (feature_drawn_lines.geography)::geometry AS geography, + (feature_drawn_lines.geography)::geometry AS line_geography, + null::integer AS signal_id, + feature_drawn_lines.length_feet + FROM feature_drawn_lines + WHERE (feature_drawn_lines.is_deleted = false) + ) AS feature_union + GROUP BY feature_union.component_id +), + +subcomponents AS ( + SELECT + mpcs.project_component_id, + string_agg(ms.subcomponent_name, ', '::text) AS subcomponents + FROM ( + moped_proj_components_subcomponents AS mpcs + LEFT JOIN moped_subcomponents AS ms ON ((mpcs.subcomponent_id = ms.subcomponent_id)) + ) + WHERE (mpcs.is_deleted = false) + GROUP BY mpcs.project_component_id +), + +component_tags AS ( + SELECT + mpct.project_component_id, + string_agg(((mct.type || ' - '::text) || mct.name), ', '::text) AS component_tags + FROM ( + moped_proj_component_tags AS mpct + LEFT JOIN moped_component_tags AS mct ON ((mpct.component_tag_id = mct.id)) + ) + WHERE (mpct.is_deleted = false) + GROUP BY mpct.project_component_id +) + +SELECT + mpc.project_id, + comp_geography.project_component_id, + comp_geography.feature_ids, + mpc.component_id, + comp_geography.geometry, + comp_geography.line_geometry, + comp_geography.signal_ids, + council_districts.council_districts, + 'placeholder text'::text AS council_districts_searchable, + ( + NOT coalesce( + ((council_districts.council_districts IS null) OR (council_districts.council_districts = ''::text)), false + ) + ) AS is_within_city_limits, + comp_geography.length_feet_total, + round(((comp_geography.length_feet_total)::numeric / (5280)::numeric), 2) AS length_miles_total, + mc.component_name, + mc.component_subtype, + mc.component_name_full, + 'placeholder text'::text AS component_categories, + subcomponents.subcomponents AS component_subcomponents, + work_types.work_types AS component_work_types, + component_tags.component_tags, + mpc.description AS component_description, + mpc.interim_project_component_id, + mpc.completion_date, + coalesce(mpc.completion_date, plv.substantial_completion_date) AS substantial_completion_date, + '2024-01-01T00:00:00-06:00'::text AS substantial_completion_date_estimated, + mpc.srts_id, + mpc.location_description AS component_location_description, + plv.project_name, + 'placeholder text'::text AS project_name_descriptor, + 'placeholder text'::text AS project_name_with_descriptor, + plv.project_description, + plv.ecapris_subproject_id, + plv.project_website, + plv.updated_at AS project_updated_at, + mpc.phase_id AS component_phase_id, + mph.phase_name AS component_phase_name, + mph.phase_name_simple AS component_phase_name_simple, + current_phase.phase_id AS project_phase_id, + current_phase.phase_name AS project_phase_name, + current_phase.phase_name_simple AS project_phase_name_simple, + coalesce(mph.phase_name, current_phase.phase_name) AS current_phase_name, + coalesce(mph.phase_name_simple, current_phase.phase_name_simple) AS current_phase_name_simple, + plv.project_team_members, + plv.project_sponsor, + plv.project_lead, + plv.public_process_status, + plv.interim_project_id, + plv.project_partners, + plv.task_order_names, + plv.funding_source_name, + plv.type_name, + plv.project_status_update, + plv.project_status_update_date_created, + plv.construction_start_date, + plv.completion_end_date, + plv.project_inspector, + plv.project_designer, + plv.project_tags, + plv.workgroup_contractors, + plv.contract_numbers, + plv.parent_project_id, + plv.parent_project_name, + plv.parent_project_url, + 'placeholder text'::text AS parent_project_name_with_descriptor, + 'placeholder text'::text AS related_project_ids, + 'placeholder text'::text AS related_project_ids_searchable, + plv.knack_project_id AS knack_data_tracker_project_record_id, + plv.project_url, + ((plv.project_url || '?tab=map&project_component_id='::text) || (mpc.project_component_id)::text) AS component_url, + plv.project_development_status, + plv.project_development_status_date, + plv.project_development_status_date_calendar_year, + plv.project_development_status_date_calendar_year_month, + plv.project_development_status_date_calendar_year_month_numeric, + plv.project_development_status_date_calendar_year_quarter, + plv.project_development_status_date_fiscal_year, + plv.project_development_status_date_fiscal_year_quarter, + plv.added_by AS project_added_by +FROM (((((((((moped_proj_components AS mpc + LEFT JOIN comp_geography ON ((mpc.project_component_id = comp_geography.project_component_id)) +) +LEFT JOIN council_districts ON ((mpc.project_component_id = council_districts.project_component_id)) +) +LEFT JOIN subcomponents ON ((mpc.project_component_id = subcomponents.project_component_id)) +) +LEFT JOIN work_types ON ((mpc.project_component_id = work_types.project_component_id)) +) +LEFT JOIN component_tags ON ((mpc.project_component_id = component_tags.project_component_id)) +) +LEFT JOIN project_list_view AS plv ON ((mpc.project_id = plv.project_id)) +) +LEFT JOIN current_phase_view AS current_phase ON ((mpc.project_id = current_phase.project_id)) +) +LEFT JOIN moped_phases AS mph ON ((mpc.phase_id = mph.phase_id)) +) +LEFT JOIN moped_components AS mc ON ((mpc.component_id = mc.component_id)) +) +WHERE ((mpc.is_deleted = false) AND (plv.is_deleted = false)); \ No newline at end of file diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql index ad88137e1e..586e354207 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql @@ -314,3 +314,227 @@ GROUP BY work_activities.task_order_names_short, work_activities.task_orders, districts.council_districts; + + + + + + + +CREATE VIEW component_arcgis_online_view AS +WITH work_types AS ( + SELECT + mpcwt.project_component_id, + string_agg(mwt.name, ', '::text) AS work_types + FROM ( + moped_proj_component_work_types AS mpcwt + LEFT JOIN moped_work_types AS mwt ON ((mpcwt.work_type_id = mwt.id)) + ) + WHERE (mpcwt.is_deleted = false) + GROUP BY mpcwt.project_component_id +), + +council_districts AS ( + SELECT + features.component_id AS project_component_id, + string_agg(DISTINCT (features_council_districts.council_district_id)::text, ', '::text) AS council_districts + FROM ( + features_council_districts + LEFT JOIN features ON ((features_council_districts.feature_id = features.id)) + ) + WHERE (features.is_deleted = false) + GROUP BY features.component_id +), + +comp_geography AS ( + SELECT + feature_union.component_id AS project_component_id, + string_agg(DISTINCT (feature_union.id)::text, ', '::text) AS feature_ids, + (st_asgeojson(st_union(array_agg(feature_union.geography))))::json AS geometry, + (st_asgeojson(st_union(array_agg(feature_union.line_geography))))::json AS line_geometry, + string_agg(DISTINCT (feature_union.signal_id)::text, ', '::text) AS signal_ids, + sum(feature_union.length_feet) AS length_feet_total + FROM ( + SELECT + feature_signals.id, + feature_signals.component_id, + (feature_signals.geography)::geometry AS geography, + st_exteriorring((st_buffer(feature_signals.geography, (7)::double precision))::geometry) AS line_geography, + feature_signals.signal_id, + null::integer AS length_feet + FROM feature_signals + WHERE (feature_signals.is_deleted = false) + UNION ALL + SELECT + feature_street_segments.id, + feature_street_segments.component_id, + (feature_street_segments.geography)::geometry AS geography, + (feature_street_segments.geography)::geometry AS line_geography, + null::integer AS signal_id, + feature_street_segments.length_feet + FROM feature_street_segments + WHERE (feature_street_segments.is_deleted = false) + UNION ALL + SELECT + feature_intersections.id, + feature_intersections.component_id, + (feature_intersections.geography)::geometry AS geography, + st_exteriorring( + (st_buffer(feature_intersections.geography, (7)::double precision))::geometry + ) AS line_geography, + null::integer AS signal_id, + null::integer AS length_feet + FROM feature_intersections + WHERE (feature_intersections.is_deleted = false) + UNION ALL + SELECT + feature_drawn_points.id, + feature_drawn_points.component_id, + (feature_drawn_points.geography)::geometry AS geography, + st_exteriorring( + (st_buffer(feature_drawn_points.geography, (7)::double precision))::geometry + ) AS line_geography, + null::integer AS signal_id, + null::integer AS length_feet + FROM feature_drawn_points + WHERE (feature_drawn_points.is_deleted = false) + UNION ALL + SELECT + feature_drawn_lines.id, + feature_drawn_lines.component_id, + (feature_drawn_lines.geography)::geometry AS geography, + (feature_drawn_lines.geography)::geometry AS line_geography, + null::integer AS signal_id, + feature_drawn_lines.length_feet + FROM feature_drawn_lines + WHERE (feature_drawn_lines.is_deleted = false) + ) AS feature_union + GROUP BY feature_union.component_id +), + +subcomponents AS ( + SELECT + mpcs.project_component_id, + string_agg(ms.subcomponent_name, ', '::text) AS subcomponents + FROM ( + moped_proj_components_subcomponents AS mpcs + LEFT JOIN moped_subcomponents AS ms ON ((mpcs.subcomponent_id = ms.subcomponent_id)) + ) + WHERE (mpcs.is_deleted = false) + GROUP BY mpcs.project_component_id +), + +component_tags AS ( + SELECT + mpct.project_component_id, + string_agg(((mct.type || ' - '::text) || mct.name), ', '::text) AS component_tags + FROM ( + moped_proj_component_tags AS mpct + LEFT JOIN moped_component_tags AS mct ON ((mpct.component_tag_id = mct.id)) + ) + WHERE (mpct.is_deleted = false) + GROUP BY mpct.project_component_id +) + +SELECT + mpc.project_id, + comp_geography.project_component_id, + comp_geography.feature_ids, + mpc.component_id, + comp_geography.geometry, + comp_geography.line_geometry, + comp_geography.signal_ids, + council_districts.council_districts, + 'placeholder text'::text AS council_districts_searchable, + ( + NOT coalesce( + ((council_districts.council_districts IS null) OR (council_districts.council_districts = ''::text)), false + ) + ) AS is_within_city_limits, + comp_geography.length_feet_total, + round(((comp_geography.length_feet_total)::numeric / (5280)::numeric), 2) AS length_miles_total, + mc.component_name, + mc.component_subtype, + mc.component_name_full, + 'placeholder text'::text AS component_categories, + subcomponents.subcomponents AS component_subcomponents, + work_types.work_types AS component_work_types, + component_tags.component_tags, + mpc.description AS component_description, + mpc.interim_project_component_id, + mpc.completion_date, + coalesce(mpc.completion_date, plv.substantial_completion_date) AS substantial_completion_date, + '2024-01-01T00:00:00-06:00'::text AS substantial_completion_date_estimated, + mpc.srts_id, + mpc.location_description AS component_location_description, + plv.project_name, + 'placeholder text'::text AS project_name_descriptor, + 'placeholder text'::text AS project_name_with_descriptor, + plv.project_description, + plv.ecapris_subproject_id, + plv.project_website, + plv.updated_at AS project_updated_at, + mpc.phase_id AS component_phase_id, + mph.phase_name AS component_phase_name, + mph.phase_name_simple AS component_phase_name_simple, + current_phase.phase_id AS project_phase_id, + current_phase.phase_name AS project_phase_name, + current_phase.phase_name_simple AS project_phase_name_simple, + coalesce(mph.phase_name, current_phase.phase_name) AS current_phase_name, + coalesce(mph.phase_name_simple, current_phase.phase_name_simple) AS current_phase_name_simple, + plv.project_team_members, + plv.project_sponsor, + plv.project_lead, + plv.public_process_status, + plv.interim_project_id, + plv.project_partners, + plv.task_order_names, + plv.funding_source_name, + plv.type_name, + plv.project_status_update, + plv.project_status_update_date_created, + plv.construction_start_date, + plv.completion_end_date, + plv.project_inspector, + plv.project_designer, + plv.project_tags, + plv.workgroup_contractors, + plv.contract_numbers, + plv.parent_project_id, + plv.parent_project_name, + plv.parent_project_url, + 'placeholder text'::text AS parent_project_name_with_descriptor, + 'placeholder text'::text AS related_project_ids, + 'placeholder text'::text AS related_project_ids_searchable, + plv.knack_project_id AS knack_data_tracker_project_record_id, + plv.project_url, + ((plv.project_url || '?tab=map&project_component_id='::text) || (mpc.project_component_id)::text) AS component_url, + plv.project_development_status, + plv.project_development_status_date, + plv.project_development_status_date_calendar_year, + plv.project_development_status_date_calendar_year_month, + plv.project_development_status_date_calendar_year_month_numeric, + plv.project_development_status_date_calendar_year_quarter, + plv.project_development_status_date_fiscal_year, + plv.project_development_status_date_fiscal_year_quarter, + plv.added_by AS project_added_by +FROM (((((((((moped_proj_components AS mpc + LEFT JOIN comp_geography ON ((mpc.project_component_id = comp_geography.project_component_id)) +) +LEFT JOIN council_districts ON ((mpc.project_component_id = council_districts.project_component_id)) +) +LEFT JOIN subcomponents ON ((mpc.project_component_id = subcomponents.project_component_id)) +) +LEFT JOIN work_types ON ((mpc.project_component_id = work_types.project_component_id)) +) +LEFT JOIN component_tags ON ((mpc.project_component_id = component_tags.project_component_id)) +) +LEFT JOIN project_list_view AS plv ON ((mpc.project_id = plv.project_id)) +) +LEFT JOIN current_phase_view AS current_phase ON ((mpc.project_id = current_phase.project_id)) +) +LEFT JOIN moped_phases AS mph ON ((mpc.phase_id = mph.phase_id)) +) +LEFT JOIN moped_components AS mc ON ((mpc.component_id = mc.component_id)) +) +WHERE ((mpc.is_deleted = false) AND (plv.is_deleted = false)); From ae7e1d0205f4cd8b42ab41b3c140ecd1026f254a Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 8 Feb 2024 12:31:48 -0600 Subject: [PATCH 45/76] no conditional operations; we're being explicit here --- .../down.sql | 2 +- .../1707416196011_add_council_districts_to_project_view/up.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql index ed624bab2f..a1b9fd8f07 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql @@ -1,7 +1,7 @@ -- no "IF EXISTS" because if it doesn't exist, we have bigger problems and we want this to bail DROP VIEW public.project_list_view CASCADE; -CREATE OR REPLACE VIEW public.project_list_view +CREATE VIEW public.project_list_view AS WITH project_person_list_lookup AS ( SELECT mpp.project_id, diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql index 586e354207..e345ce76a3 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql @@ -1,7 +1,7 @@ -- no "IF EXISTS" because if it doesn't exist, we have bigger problems and we want this to bail DROP VIEW public.project_list_view CASCADE; -CREATE OR REPLACE VIEW public.project_list_view AS +CREATE VIEW public.project_list_view AS WITH project_person_list_lookup AS ( SELECT mpp.project_id, From 1048322ba0e03c25cce66496cd146bbf1bedb470 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 8 Feb 2024 12:36:09 -0600 Subject: [PATCH 46/76] expose new column via the graphql endpoint --- moped-database/metadata/tables.yaml | 129 ++++++++++++++-------------- 1 file changed, 66 insertions(+), 63 deletions(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index 073057a62b..fdf0068168 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -5084,163 +5084,166 @@ - role: moped-admin permission: columns: - - is_deleted - - completion_end_date - - construction_start_date - - substantial_completion_date - - interim_project_id - - parent_project_id - - project_id - - project_development_status_date_calendar_year - - project_development_status_date_fiscal_year - - project_feature - - children_project_ids - - task_orders - added_by + - children_project_ids + - completion_end_date - components + - construction_start_date - contract_numbers + - council_districts - current_phase - current_phase_key - current_phase_simple + - date_added - ecapris_subproject_id - funding_source_name + - interim_project_id + - is_deleted - knack_project_id + - parent_project_id - parent_project_name - parent_project_url - project_description - project_designer + - project_development_status + - project_development_status_date + - project_development_status_date_calendar_year + - project_development_status_date_calendar_year_month + - project_development_status_date_calendar_year_month_numeric + - project_development_status_date_calendar_year_quarter + - project_development_status_date_fiscal_year + - project_development_status_date_fiscal_year_quarter + - project_feature + - project_id - project_inspector - project_lead - project_name - project_partners - project_sponsor - project_status_update + - project_status_update_date_created - project_tags - project_team_members - project_url - project_website - public_process_status + - substantial_completion_date - task_order_names - task_order_names_short + - task_orders - type_name - - workgroup_contractors - - project_development_status - - project_development_status_date - - project_development_status_date_calendar_year_month - - project_development_status_date_calendar_year_month_numeric - - project_development_status_date_calendar_year_quarter - - project_development_status_date_fiscal_year_quarter - - date_added - - project_status_update_date_created - updated_at + - workgroup_contractors filter: {} allow_aggregations: true - role: moped-editor permission: columns: - - is_deleted - - completion_end_date - - construction_start_date - - substantial_completion_date - - interim_project_id - - parent_project_id - - project_id - - project_development_status_date_calendar_year - - project_development_status_date_fiscal_year - - project_feature - - children_project_ids - - task_orders - added_by + - children_project_ids + - completion_end_date - components + - construction_start_date - contract_numbers + - council_districts - current_phase - current_phase_key - current_phase_simple + - date_added - ecapris_subproject_id - funding_source_name + - interim_project_id + - is_deleted - knack_project_id + - parent_project_id - parent_project_name - parent_project_url - project_description - project_designer + - project_development_status + - project_development_status_date + - project_development_status_date_calendar_year + - project_development_status_date_calendar_year_month + - project_development_status_date_calendar_year_month_numeric + - project_development_status_date_calendar_year_quarter + - project_development_status_date_fiscal_year + - project_development_status_date_fiscal_year_quarter + - project_feature + - project_id - project_inspector - project_lead - project_name - project_partners - project_sponsor - project_status_update + - project_status_update_date_created - project_tags - project_team_members - project_url - project_website - public_process_status + - substantial_completion_date - task_order_names - task_order_names_short + - task_orders - type_name - - workgroup_contractors - - project_development_status - - project_development_status_date - - project_development_status_date_calendar_year_month - - project_development_status_date_calendar_year_month_numeric - - project_development_status_date_calendar_year_quarter - - project_development_status_date_fiscal_year_quarter - - date_added - - project_status_update_date_created - updated_at + - workgroup_contractors filter: {} allow_aggregations: true - role: moped-viewer permission: columns: - - is_deleted - - completion_end_date - - construction_start_date - - substantial_completion_date - - interim_project_id - - parent_project_id - - project_id - - project_development_status_date_calendar_year - - project_development_status_date_fiscal_year - - project_feature - - children_project_ids - - task_orders - added_by + - children_project_ids + - completion_end_date - components + - construction_start_date - contract_numbers + - council_districts - current_phase - current_phase_key - current_phase_simple + - date_added - ecapris_subproject_id - funding_source_name + - interim_project_id + - is_deleted - knack_project_id + - parent_project_id - parent_project_name - parent_project_url - project_description - project_designer + - project_development_status + - project_development_status_date + - project_development_status_date_calendar_year + - project_development_status_date_calendar_year_month + - project_development_status_date_calendar_year_month_numeric + - project_development_status_date_calendar_year_quarter + - project_development_status_date_fiscal_year + - project_development_status_date_fiscal_year_quarter + - project_feature + - project_id - project_inspector - project_lead - project_name - project_partners - project_sponsor - project_status_update + - project_status_update_date_created - project_tags - project_team_members - project_url - project_website - public_process_status + - substantial_completion_date - task_order_names - task_order_names_short + - task_orders - type_name - - workgroup_contractors - - project_development_status - - project_development_status_date - - project_development_status_date_calendar_year_month - - project_development_status_date_calendar_year_month_numeric - - project_development_status_date_calendar_year_quarter - - project_development_status_date_fiscal_year_quarter - - date_added - - project_status_update_date_created - updated_at + - workgroup_contractors filter: {} allow_aggregations: true - table: From a123d94c4016b82a70dca5de4b7b6b83c1e0a48b Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Fri, 9 Feb 2024 17:04:16 -0600 Subject: [PATCH 47/76] Add a self and self and child column. --- moped-database/views/project_list_view.sql | 54 ++++++++++++++++------ 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/moped-database/views/project_list_view.sql b/moped-database/views/project_list_view.sql index 7457331c20..6b543d48f6 100644 --- a/moped-database/views/project_list_view.sql +++ b/moped-database/views/project_list_view.sql @@ -84,18 +84,42 @@ moped_proj_components_subtypes AS ( ), project_district_association AS ( - SELECT - flattened_project_council_map.project_id, - array_agg(DISTINCT flattened_project_council_map.single_districts) AS council_districts - FROM ( + + WITH + project_council_district_map AS ( + SELECT DISTINCT + project_id, + unnest(council_districts) AS council_district + FROM project_geography + ), + + parent_child_project_map AS ( SELECT - components.project_id, - unnest(features.council_districts) AS single_districts - FROM moped_proj_components AS components - INNER JOIN uniform_features AS features ON (components.component_id = features.component_id) - WHERE components.is_deleted IS false - ) AS flattened_project_council_map - GROUP BY flattened_project_council_map.project_id + parent_projects.project_id, + unnest( + ARRAY[parent_projects.project_id] + || array_agg(child_projects.project_id) + ) AS self_and_children_project_ids + FROM moped_project AS parent_projects + LEFT JOIN moped_project AS child_projects ON parent_projects.project_id = child_projects.parent_project_id + GROUP BY parent_projects.project_id + ORDER BY parent_projects.project_id ASC + ) + + SELECT + projects.project_id, + array_agg(DISTINCT project_districts.council_district) FILTER ( + WHERE project_districts.council_district IS NOT null + ) AS self_alone_council_districts, + array_agg(DISTINCT districts.council_district) FILTER ( + WHERE districts.council_district IS NOT null + ) AS self_and_children_council_districts + FROM parent_child_project_map AS projects + LEFT JOIN + project_council_district_map AS districts + ON (projects.self_and_children_project_ids = districts.project_id) + LEFT JOIN project_council_district_map AS project_districts ON (projects.project_id = project_districts.project_id) + GROUP BY projects.project_id ) SELECT @@ -251,7 +275,8 @@ SELECT ) AS project_tags, concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, mpcs.components, - districts.council_districts + districts.self_alone_council_districts, + districts.self_and_children_council_districts FROM moped_project AS mp LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id @@ -266,7 +291,7 @@ LEFT JOIN current_phase_view AS current_phase ON mp.project_id = current_phase.p LEFT JOIN moped_public_process_statuses AS mpps ON mp.public_process_status_id = mpps.id LEFT JOIN child_project_lookup AS cpl ON mp.project_id = cpl.parent_id LEFT JOIN moped_proj_components_subtypes AS mpcs ON mp.project_id = mpcs.project_id -LEFT JOIN project_district_association AS districts ON (mp.project_id = districts.project_id) +LEFT JOIN project_district_association AS districts ON mp.project_id = districts.project_id LEFT JOIN LATERAL ( SELECT @@ -310,4 +335,5 @@ GROUP BY work_activities.task_order_names, work_activities.task_order_names_short, work_activities.task_orders, - districts.council_districts; + districts.self_alone_council_districts, + districts.self_and_children_council_districts; From 70051783a441dab11d5e4ce443e5bc1533a1275f Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Fri, 9 Feb 2024 17:41:06 -0600 Subject: [PATCH 48/76] add revised sql into the up migration --- .../up.sql | 58 +++++++++++++------ 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql index e345ce76a3..6e3d35a025 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql @@ -1,7 +1,7 @@ -- no "IF EXISTS" because if it doesn't exist, we have bigger problems and we want this to bail DROP VIEW public.project_list_view CASCADE; -CREATE VIEW public.project_list_view AS +CREATE OR REPLACE VIEW public.project_list_view AS WITH project_person_list_lookup AS ( SELECT mpp.project_id, @@ -87,18 +87,42 @@ moped_proj_components_subtypes AS ( ), project_district_association AS ( - SELECT - flattened_project_council_map.project_id, - array_agg(DISTINCT flattened_project_council_map.single_districts) AS council_districts - FROM ( + + WITH + project_council_district_map AS ( + SELECT DISTINCT + project_id, + unnest(council_districts) AS council_district + FROM project_geography + ), + + parent_child_project_map AS ( SELECT - components.project_id, - unnest(features.council_districts) AS single_districts - FROM moped_proj_components AS components - INNER JOIN uniform_features AS features ON (components.component_id = features.component_id) - WHERE components.is_deleted IS false - ) AS flattened_project_council_map - GROUP BY flattened_project_council_map.project_id + parent_projects.project_id, + unnest( + ARRAY[parent_projects.project_id] + || array_agg(child_projects.project_id) + ) AS self_and_children_project_ids + FROM moped_project AS parent_projects + LEFT JOIN moped_project AS child_projects ON parent_projects.project_id = child_projects.parent_project_id + GROUP BY parent_projects.project_id + ORDER BY parent_projects.project_id ASC + ) + + SELECT + projects.project_id, + array_agg(DISTINCT project_districts.council_district) FILTER ( + WHERE project_districts.council_district IS NOT null + ) AS self_alone_council_districts, + array_agg(DISTINCT districts.council_district) FILTER ( + WHERE districts.council_district IS NOT null + ) AS self_and_children_council_districts + FROM parent_child_project_map AS projects + LEFT JOIN + project_council_district_map AS districts + ON (projects.self_and_children_project_ids = districts.project_id) + LEFT JOIN project_council_district_map AS project_districts ON (projects.project_id = project_districts.project_id) + GROUP BY projects.project_id ) SELECT @@ -254,7 +278,8 @@ SELECT ) AS project_tags, concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, mpcs.components, - districts.council_districts + districts.self_alone_council_districts, + districts.self_and_children_council_districts FROM moped_project AS mp LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id @@ -269,7 +294,7 @@ LEFT JOIN current_phase_view AS current_phase ON mp.project_id = current_phase.p LEFT JOIN moped_public_process_statuses AS mpps ON mp.public_process_status_id = mpps.id LEFT JOIN child_project_lookup AS cpl ON mp.project_id = cpl.parent_id LEFT JOIN moped_proj_components_subtypes AS mpcs ON mp.project_id = mpcs.project_id -LEFT JOIN project_district_association AS districts ON (mp.project_id = districts.project_id) +LEFT JOIN project_district_association AS districts ON mp.project_id = districts.project_id LEFT JOIN LATERAL ( SELECT @@ -313,9 +338,8 @@ GROUP BY work_activities.task_order_names, work_activities.task_order_names_short, work_activities.task_orders, - districts.council_districts; - - + districts.self_alone_council_districts, + districts.self_and_children_council_districts; From f840ef5b7835aea60798cf27ee24a503e3a14095 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Fri, 9 Feb 2024 17:41:23 -0600 Subject: [PATCH 49/76] update the metadata to recognize these columns --- moped-database/metadata/tables.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index fdf0068168..fb098e4ad0 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -5090,7 +5090,8 @@ - components - construction_start_date - contract_numbers - - council_districts + - self_alone_council_districts + - self_and_children_council_districts - current_phase - current_phase_key - current_phase_simple @@ -5145,7 +5146,8 @@ - components - construction_start_date - contract_numbers - - council_districts + - self_alone_council_districts + - self_and_children_council_districts - current_phase - current_phase_key - current_phase_simple @@ -5200,7 +5202,8 @@ - components - construction_start_date - contract_numbers - - council_districts + - self_alone_council_districts + - self_and_children_council_districts - current_phase - current_phase_key - current_phase_simple From 00dd4ecffc6b70655feea3820a3a425efd340f89 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Fri, 9 Feb 2024 17:45:32 -0600 Subject: [PATCH 50/76] EOF linefeed --- .../down.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql index a1b9fd8f07..e2159c8c2b 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/down.sql @@ -516,4 +516,4 @@ LEFT JOIN moped_phases AS mph ON ((mpc.phase_id = mph.phase_id)) ) LEFT JOIN moped_components AS mc ON ((mpc.component_id = mc.component_id)) ) -WHERE ((mpc.is_deleted = false) AND (plv.is_deleted = false)); \ No newline at end of file +WHERE ((mpc.is_deleted = false) AND (plv.is_deleted = false)); From bad6b966ec6d2b4837d56ae8951cc2e301e05d05 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 12 Feb 2024 10:13:17 -0600 Subject: [PATCH 51/76] Add self_and_children_council_districts column to project list view --- .../projectsListView/ProjectsListViewQueryConf.js | 6 ++++++ .../src/views/projects/projectsListView/helpers.js | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js b/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js index 13ae14b3e4..b8fbb6ff38 100644 --- a/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js +++ b/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js @@ -290,6 +290,12 @@ export const PROJECT_LIST_VIEW_QUERY_CONFIG = { defaultHidden: true, showInTable: true, }, + self_and_children_council_districts: { + type: "array", + sortable: false, + defaultHidden: false, + showInTable: true, + }, }, }; diff --git a/moped-editor/src/views/projects/projectsListView/helpers.js b/moped-editor/src/views/projects/projectsListView/helpers.js index 0e594bc90d..b9752ae969 100644 --- a/moped-editor/src/views/projects/projectsListView/helpers.js +++ b/moped-editor/src/views/projects/projectsListView/helpers.js @@ -452,6 +452,19 @@ export const useColumns = ({ hiddenColumns }) => { }, emptyValue: "-", }, + { + title: "Council districts", + field: "self_and_children_council_districts", + hidden: hiddenColumns["self_and_children_council_districts"], + render: (entry) => { + if (entry.self_and_children_council_districts.length > 0) { + return entry.self_and_children_council_districts.join(", "); + } else { + return - ; + } + }, + emptyValue: "-", + }, ], [hiddenColumns, queryString] ); From a980803ee5e0ff98dabe838804096390b2c5f8f8 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 12 Feb 2024 10:16:03 -0600 Subject: [PATCH 52/76] Hide self_and_children_council_districts column by default in project list view --- .../projects/projectsListView/ProjectsListViewQueryConf.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js b/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js index b8fbb6ff38..ca83fa5add 100644 --- a/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js +++ b/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js @@ -293,7 +293,7 @@ export const PROJECT_LIST_VIEW_QUERY_CONFIG = { self_and_children_council_districts: { type: "array", sortable: false, - defaultHidden: false, + defaultHidden: true, showInTable: true, }, }, From 6f586dc7ea2622f96e559f030442b4288f3bdcac Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 12 Feb 2024 13:50:36 -0600 Subject: [PATCH 53/76] Add 'is blank' and 'is not blank' operators for council districts --- .../components/GridTable/FiltersCommonOperators.js | 14 ++++++++++++++ .../ProjectsListViewFiltersConf.js | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/moped-editor/src/components/GridTable/FiltersCommonOperators.js b/moped-editor/src/components/GridTable/FiltersCommonOperators.js index 708875d09f..e60a12d1cc 100644 --- a/moped-editor/src/components/GridTable/FiltersCommonOperators.js +++ b/moped-editor/src/components/GridTable/FiltersCommonOperators.js @@ -168,4 +168,18 @@ export const FiltersCommonOperators = { type: "array", envelope: "false", }, + council_districts_array_is_null: { + operator: "_is_null", + label: "No", + description: "Project's has no mapped components in council districts", + type: "array", + envelope: "true", + }, + council_districts_array_is_not_null: { + operator: "_is_null", + label: "Yes", + description: "Project's has components in council districts", + type: "array", + envelope: "false", + }, }; diff --git a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js index e2ba654c90..5901ab3af2 100644 --- a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js +++ b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js @@ -24,6 +24,8 @@ export const OPERATORS_WITHOUT_SEARCH_VALUES = [ "subprojects_array_is_not_null", "string_is_blank", "string_is_not_blank", + "council_districts_array_is_null", + "council_districts_array_is_not_null", ]; /** @@ -414,6 +416,16 @@ export const PROJECT_LIST_VIEW_FILTERS_CONFIG = { "string_is_not_null", ], }, + { + name: "self_and_children_council_districts", + label: "Council Districts", + placeholder: "District", + type: "array", + operators: [ + "council_districts_array_is_null", + "council_districts_array_is_not_null", + ], + }, ], operators: { From a72759d723867e305163e08692828af60bc09cc9 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 12 Feb 2024 16:17:40 -0600 Subject: [PATCH 54/76] Update descriptions in FiltersCommonOperators.js --- .../src/components/GridTable/FiltersCommonOperators.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-editor/src/components/GridTable/FiltersCommonOperators.js b/moped-editor/src/components/GridTable/FiltersCommonOperators.js index e60a12d1cc..0e110c75d1 100644 --- a/moped-editor/src/components/GridTable/FiltersCommonOperators.js +++ b/moped-editor/src/components/GridTable/FiltersCommonOperators.js @@ -171,14 +171,14 @@ export const FiltersCommonOperators = { council_districts_array_is_null: { operator: "_is_null", label: "No", - description: "Project's has no mapped components in council districts", + description: "Project has no mapped components in council districts", type: "array", envelope: "true", }, council_districts_array_is_not_null: { operator: "_is_null", label: "Yes", - description: "Project's has components in council districts", + description: "Project has components in council districts", type: "array", envelope: "false", }, From ee4faca1176194a8f4fbc88176c7f9fbd9c8239b Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 12 Feb 2024 16:17:52 -0600 Subject: [PATCH 55/76] Add council_districts_array_contains operator to FiltersCommonOperators.js --- .../src/components/GridTable/FiltersCommonOperators.js | 7 +++++++ .../projectsListView/ProjectsListViewFiltersConf.js | 1 + 2 files changed, 8 insertions(+) diff --git a/moped-editor/src/components/GridTable/FiltersCommonOperators.js b/moped-editor/src/components/GridTable/FiltersCommonOperators.js index 0e110c75d1..c53f753189 100644 --- a/moped-editor/src/components/GridTable/FiltersCommonOperators.js +++ b/moped-editor/src/components/GridTable/FiltersCommonOperators.js @@ -182,4 +182,11 @@ export const FiltersCommonOperators = { type: "array", envelope: "false", }, + council_districts_array_contains: { + operator: "_contains", + label: "Contains", + description: "Project has components in a given council district", + type: "array", + envelope: false, + }, }; diff --git a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js index 5901ab3af2..b1443b940b 100644 --- a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js +++ b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js @@ -424,6 +424,7 @@ export const PROJECT_LIST_VIEW_FILTERS_CONFIG = { operators: [ "council_districts_array_is_null", "council_districts_array_is_not_null", + "council_districts_array_contains", ], }, ], From fbb852c6271cc860a19ed1e2433fc418b622d6c6 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 12 Feb 2024 17:19:19 -0600 Subject: [PATCH 56/76] replace `in` statement with conditional structure; add array formatting. This took me way longer than it should. The stringification for the graphql query is too tricky. --- .../useProjectListViewQuery/useAdvancedSearch.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch.js b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch.js index c5b96b3a5b..cb382cc73e 100644 --- a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch.js +++ b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch.js @@ -73,7 +73,17 @@ const makeAdvancedSearchWhereFilters = (filters) => // If it is a number or boolean, it does not need quotation marks // Otherwise, add quotation marks for the query to identify as string - value = type in ["number", "boolean"] ? value : `"${value}"`; + if (["number", "boolean"].includes(type)) { + // eslint-disable-next-line no-self-assign + value = value; + } else if (type === "array") { + // a virtue of this stringification of an array, we actually support + // comma delimited lists of districts and `graphql-engine` will do the + // correct set algebra + value = `[${value}]`; + } else { + value = `"${value}"`; + } } else { // We don't have a value return null; From 700d3731e3a958d4954ebad2850963712425e8d3 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 12 Feb 2024 17:29:24 -0600 Subject: [PATCH 57/76] Add new eq() filter for council districts array --- .../src/components/GridTable/FiltersCommonOperators.js | 7 +++++++ .../projectsListView/ProjectsListViewFiltersConf.js | 1 + 2 files changed, 8 insertions(+) diff --git a/moped-editor/src/components/GridTable/FiltersCommonOperators.js b/moped-editor/src/components/GridTable/FiltersCommonOperators.js index c53f753189..9e8e0ce4f2 100644 --- a/moped-editor/src/components/GridTable/FiltersCommonOperators.js +++ b/moped-editor/src/components/GridTable/FiltersCommonOperators.js @@ -189,4 +189,11 @@ export const FiltersCommonOperators = { type: "array", envelope: false, }, + council_districts_array_is: { + operator: "_eq", + label: "Is", + description: "Project has components in, and only in, specific council districts", + type: "array", + envelope: false, + }, }; diff --git a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js index b1443b940b..40767fad9c 100644 --- a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js +++ b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js @@ -425,6 +425,7 @@ export const PROJECT_LIST_VIEW_FILTERS_CONFIG = { "council_districts_array_is_null", "council_districts_array_is_not_null", "council_districts_array_contains", + "council_districts_array_is", ], }, ], From c0c1a5a64f5202fed8fd5d676c24e5323d61e054 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 13 Feb 2024 13:15:56 -0600 Subject: [PATCH 58/76] Update field names based on feedback --- moped-database/metadata/tables.yaml | 12 ++++++------ .../up.sql | 12 ++++++------ moped-database/views/project_list_view.sql | 12 ++++++------ .../projectsListView/ProjectsListViewQueryConf.js | 2 +- .../src/views/projects/projectsListView/helpers.js | 8 ++++---- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index fb098e4ad0..6c3a481822 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -5090,8 +5090,8 @@ - components - construction_start_date - contract_numbers - - self_alone_council_districts - - self_and_children_council_districts + - project_council_districts + - project_and_child_project_council_districts - current_phase - current_phase_key - current_phase_simple @@ -5146,8 +5146,8 @@ - components - construction_start_date - contract_numbers - - self_alone_council_districts - - self_and_children_council_districts + - project_council_districts + - project_and_child_project_council_districts - current_phase - current_phase_key - current_phase_simple @@ -5202,8 +5202,8 @@ - components - construction_start_date - contract_numbers - - self_alone_council_districts - - self_and_children_council_districts + - project_council_districts + - project_and_child_project_council_districts - current_phase - current_phase_key - current_phase_simple diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql index 6e3d35a025..74f78e1507 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql @@ -113,10 +113,10 @@ project_district_association AS ( projects.project_id, array_agg(DISTINCT project_districts.council_district) FILTER ( WHERE project_districts.council_district IS NOT null - ) AS self_alone_council_districts, + ) AS project_council_districts, array_agg(DISTINCT districts.council_district) FILTER ( WHERE districts.council_district IS NOT null - ) AS self_and_children_council_districts + ) AS project_and_child_project_council_districts FROM parent_child_project_map AS projects LEFT JOIN project_council_district_map AS districts @@ -278,8 +278,8 @@ SELECT ) AS project_tags, concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, mpcs.components, - districts.self_alone_council_districts, - districts.self_and_children_council_districts + districts.project_council_districts, + districts.project_and_child_project_council_districts FROM moped_project AS mp LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id @@ -338,8 +338,8 @@ GROUP BY work_activities.task_order_names, work_activities.task_order_names_short, work_activities.task_orders, - districts.self_alone_council_districts, - districts.self_and_children_council_districts; + districts.project_council_districts, + districts.project_and_child_project_council_districts; diff --git a/moped-database/views/project_list_view.sql b/moped-database/views/project_list_view.sql index 6b543d48f6..e16c3d507e 100644 --- a/moped-database/views/project_list_view.sql +++ b/moped-database/views/project_list_view.sql @@ -110,10 +110,10 @@ project_district_association AS ( projects.project_id, array_agg(DISTINCT project_districts.council_district) FILTER ( WHERE project_districts.council_district IS NOT null - ) AS self_alone_council_districts, + ) AS project_council_districts, array_agg(DISTINCT districts.council_district) FILTER ( WHERE districts.council_district IS NOT null - ) AS self_and_children_council_districts + ) AS project_and_child_project_council_districts FROM parent_child_project_map AS projects LEFT JOIN project_council_district_map AS districts @@ -275,8 +275,8 @@ SELECT ) AS project_tags, concat(added_by_user.first_name, ' ', added_by_user.last_name) AS added_by, mpcs.components, - districts.self_alone_council_districts, - districts.self_and_children_council_districts + districts.project_council_districts, + districts.project_and_child_project_council_districts FROM moped_project AS mp LEFT JOIN project_person_list_lookup AS ppll ON mp.project_id = ppll.project_id LEFT JOIN funding_sources_lookup AS fsl ON mp.project_id = fsl.project_id @@ -335,5 +335,5 @@ GROUP BY work_activities.task_order_names, work_activities.task_order_names_short, work_activities.task_orders, - districts.self_alone_council_districts, - districts.self_and_children_council_districts; + districts.project_council_districts, + districts.project_and_child_project_council_districts; diff --git a/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js b/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js index ca83fa5add..4080efab9c 100644 --- a/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js +++ b/moped-editor/src/views/projects/projectsListView/ProjectsListViewQueryConf.js @@ -290,7 +290,7 @@ export const PROJECT_LIST_VIEW_QUERY_CONFIG = { defaultHidden: true, showInTable: true, }, - self_and_children_council_districts: { + project_and_child_project_council_districts: { type: "array", sortable: false, defaultHidden: true, diff --git a/moped-editor/src/views/projects/projectsListView/helpers.js b/moped-editor/src/views/projects/projectsListView/helpers.js index b9752ae969..9f95a7a88f 100644 --- a/moped-editor/src/views/projects/projectsListView/helpers.js +++ b/moped-editor/src/views/projects/projectsListView/helpers.js @@ -454,11 +454,11 @@ export const useColumns = ({ hiddenColumns }) => { }, { title: "Council districts", - field: "self_and_children_council_districts", - hidden: hiddenColumns["self_and_children_council_districts"], + field: "project_and_child_project_council_districts", + hidden: hiddenColumns["project_and_child_project_council_districts"], render: (entry) => { - if (entry.self_and_children_council_districts.length > 0) { - return entry.self_and_children_council_districts.join(", "); + if (entry.project_and_child_project_council_districts.length > 0) { + return entry.project_and_child_project_council_districts.join(", "); } else { return - ; } From 0ecd00b0afbb1eff6e60609cc794b768afbaaba3 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 13 Feb 2024 14:02:43 -0600 Subject: [PATCH 59/76] Add check for deleted components in project_geography view --- .../down.sql | 26 ++++++++++++++++++ .../up.sql | 27 +++++++++++++++++++ moped-database/views/project_geography.sql | 5 ++-- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 moped-database/migrations/1707416196012_add_check_for_deleted_components/down.sql create mode 100644 moped-database/migrations/1707416196012_add_check_for_deleted_components/up.sql diff --git a/moped-database/migrations/1707416196012_add_check_for_deleted_components/down.sql b/moped-database/migrations/1707416196012_add_check_for_deleted_components/down.sql new file mode 100644 index 0000000000..59336edd30 --- /dev/null +++ b/moped-database/migrations/1707416196012_add_check_for_deleted_components/down.sql @@ -0,0 +1,26 @@ +CREATE OR REPLACE VIEW public.project_geography +AS SELECT + moped_project.project_id, + uniform_features.id AS feature_id, + moped_components.component_id AS component_archtype_id, + moped_proj_components.project_component_id AS component_id, + moped_proj_components.is_deleted, + moped_project.project_name, + feature_layers.internal_table AS "table", + feature_layers.reference_layer_primary_key_column AS original_fk, + moped_components.component_name, + uniform_features.attributes, + uniform_features.geography, + uniform_features.council_districts, + uniform_features.length_feet, + uniform_features.created_at AS feature_created_at, + uniform_features.updated_at AS feature_updated_at, + uniform_features.created_by_user_id AS feature_created_by_user_id, + uniform_features.updated_by_user_id AS feature_updated_by_user_id +FROM moped_project +INNER JOIN moped_proj_components ON moped_project.project_id = moped_proj_components.project_id +INNER JOIN moped_components ON moped_proj_components.component_id = moped_components.component_id +INNER JOIN feature_layers ON moped_components.feature_layer_id = feature_layers.id +INNER JOIN uniform_features ON moped_proj_components.project_component_id = uniform_features.component_id; + +COMMENT ON VIEW public.project_geography IS 'The project_geography view merges project-specific data with the unified geographical features from the uniform_features view. It links projects with their respective geographical components, including type, attributes, and location.'; -- noqa: LT05 diff --git a/moped-database/migrations/1707416196012_add_check_for_deleted_components/up.sql b/moped-database/migrations/1707416196012_add_check_for_deleted_components/up.sql new file mode 100644 index 0000000000..02a02e229a --- /dev/null +++ b/moped-database/migrations/1707416196012_add_check_for_deleted_components/up.sql @@ -0,0 +1,27 @@ +CREATE OR REPLACE VIEW public.project_geography +AS SELECT + moped_project.project_id, + uniform_features.id AS feature_id, + moped_components.component_id AS component_archtype_id, + moped_proj_components.project_component_id AS component_id, + moped_proj_components.is_deleted, + moped_project.project_name, + feature_layers.internal_table AS "table", + feature_layers.reference_layer_primary_key_column AS original_fk, + moped_components.component_name, + uniform_features.attributes, + uniform_features.geography, + uniform_features.council_districts, + uniform_features.length_feet, + uniform_features.created_at AS feature_created_at, + uniform_features.updated_at AS feature_updated_at, + uniform_features.created_by_user_id AS feature_created_by_user_id, + uniform_features.updated_by_user_id AS feature_updated_by_user_id +FROM moped_project +INNER JOIN moped_proj_components ON moped_project.project_id = moped_proj_components.project_id +INNER JOIN moped_components ON moped_proj_components.component_id = moped_components.component_id +INNER JOIN feature_layers ON moped_components.feature_layer_id = feature_layers.id +INNER JOIN uniform_features ON moped_proj_components.project_component_id = uniform_features.component_id +WHERE moped_proj_components.is_deleted IS false; + +COMMENT ON VIEW public.project_geography IS 'The project_geography view merges project-specific data with the unified geographical features from the uniform_features view. It links projects with their respective geographical components, including type, attributes, and location.'; -- noqa: LT05 diff --git a/moped-database/views/project_geography.sql b/moped-database/views/project_geography.sql index 7561f3c505..67229b1225 100644 --- a/moped-database/views/project_geography.sql +++ b/moped-database/views/project_geography.sql @@ -23,6 +23,7 @@ FROM moped_project INNER JOIN moped_proj_components ON moped_project.project_id = moped_proj_components.project_id INNER JOIN moped_components ON moped_proj_components.component_id = moped_components.component_id INNER JOIN feature_layers ON moped_components.feature_layer_id = feature_layers.id -INNER JOIN uniform_features ON moped_proj_components.project_component_id = uniform_features.component_id; +INNER JOIN uniform_features ON moped_proj_components.project_component_id = uniform_features.component_id +WHERE moped_proj_components.is_deleted IS false; -COMMENT ON VIEW public.project_geography IS 'The project_geography view merges project-specific data with the unified geographical features from the uniform_features view. It links projects with their respective geographical components, including type, attributes, and location.'; +COMMENT ON VIEW public.project_geography IS 'The project_geography view merges project-specific data with the unified geographical features from the uniform_features view. It links projects with their respective geographical components, including type, attributes, and location.'; -- noqa: LT05 From aa478215d2be3959bbc3ded4b0119fde3280ef11 Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:27:37 -0600 Subject: [PATCH 60/76] removed a comma that was breaking the query --- .../up.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql index 4b8981e802..3fd1df880f 100644 --- a/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql +++ b/moped-database/migrations/1706746024752_alter_table_public_moped_proj_milestones_alter_column_date_added/up.sql @@ -1,9 +1,9 @@ -ALTER TABLE "public"."moped_proj_milestones" RENAME COLUMN "date_added" TO "created_at"; -ALTER TABLE "public"."moped_proj_milestones" ADD COLUMN "created_by_user_id" INTEGER +ALTER TABLE moped_proj_milestones RENAME COLUMN date_added TO created_at; +ALTER TABLE moped_proj_milestones ADD COLUMN created_by_user_id INTEGER NULL; -ALTER TABLE "public"."moped_proj_milestones" ADD COLUMN "updated_at" TIMESTAMPTZ +ALTER TABLE moped_proj_milestones ADD COLUMN updated_at TIMESTAMPTZ NULL; -ALTER TABLE "public"."moped_proj_milestones" ADD COLUMN "updated_by_user_id" INTEGER +ALTER TABLE moped_proj_milestones ADD COLUMN updated_by_user_id INTEGER NULL; COMMENT ON COLUMN moped_proj_milestones.updated_at IS 'Timestamp when the record was last updated'; From b21fafd9bac25983065e8b8f79d15032f8581909 Mon Sep 17 00:00:00 2001 From: Tilly Whitson <35410637+tillyw@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:27:43 -0600 Subject: [PATCH 61/76] removed a comma --- moped-database/metadata/tables.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index da4baa8928..59e50fce24 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -2784,7 +2784,7 @@ "description": [{"newSchema": "true"}], "operation_type": {{ $body.event.op }}, "updated_by_user_id": {{ $session_variables?['x-hasura-user-db-id'] ?? 1}} - }, + } } } method: POST From ead951d74cfc7ec582a0bbac108d5d92e0c696d4 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 13 Feb 2024 16:19:19 -0600 Subject: [PATCH 62/76] replace use of `project_geography` with a query avoiding the `union`ing in that view --- .../up.sql | 19 ++++++++++++------- moped-database/views/project_list_view.sql | 19 ++++++++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql index 74f78e1507..bc8bc410fd 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql @@ -91,9 +91,14 @@ project_district_association AS ( WITH project_council_district_map AS ( SELECT DISTINCT - project_id, - unnest(council_districts) AS council_district - FROM project_geography + moped_project.project_id, + features_council_districts.council_district_id + FROM + moped_project + LEFT JOIN moped_proj_components ON (moped_project.project_id = moped_proj_components.project_id) + LEFT JOIN features ON (moped_proj_components.project_component_id = features.component_id) + LEFT JOIN features_council_districts ON features.id = features_council_districts.feature_id + WHERE moped_proj_components.is_deleted IS false ), parent_child_project_map AS ( @@ -111,11 +116,11 @@ project_district_association AS ( SELECT projects.project_id, - array_agg(DISTINCT project_districts.council_district) FILTER ( - WHERE project_districts.council_district IS NOT null + array_agg(DISTINCT districts.council_district_id) FILTER ( + WHERE districts.council_district_id IS NOT null ) AS project_council_districts, - array_agg(DISTINCT districts.council_district) FILTER ( - WHERE districts.council_district IS NOT null + array_agg(DISTINCT districts.council_district_id) FILTER ( + WHERE districts.council_district_id IS NOT null ) AS project_and_child_project_council_districts FROM parent_child_project_map AS projects LEFT JOIN diff --git a/moped-database/views/project_list_view.sql b/moped-database/views/project_list_view.sql index e16c3d507e..9300a50495 100644 --- a/moped-database/views/project_list_view.sql +++ b/moped-database/views/project_list_view.sql @@ -88,9 +88,14 @@ project_district_association AS ( WITH project_council_district_map AS ( SELECT DISTINCT - project_id, - unnest(council_districts) AS council_district - FROM project_geography + moped_project.project_id, + features_council_districts.council_district_id + FROM + moped_project + LEFT JOIN moped_proj_components ON (moped_project.project_id = moped_proj_components.project_id) + LEFT JOIN features ON (moped_proj_components.project_component_id = features.component_id) + LEFT JOIN features_council_districts ON features.id = features_council_districts.feature_id + WHERE moped_proj_components.is_deleted IS false ), parent_child_project_map AS ( @@ -108,11 +113,11 @@ project_district_association AS ( SELECT projects.project_id, - array_agg(DISTINCT project_districts.council_district) FILTER ( - WHERE project_districts.council_district IS NOT null + array_agg(DISTINCT districts.council_district_id) FILTER ( + WHERE districts.council_district_id IS NOT null ) AS project_council_districts, - array_agg(DISTINCT districts.council_district) FILTER ( - WHERE districts.council_district IS NOT null + array_agg(DISTINCT districts.council_district_id) FILTER ( + WHERE districts.council_district_id IS NOT null ) AS project_and_child_project_council_districts FROM parent_child_project_map AS projects LEFT JOIN From 924f84f127faebb11c8273878c681d42e2ea2e57 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 13 Feb 2024 16:35:09 -0600 Subject: [PATCH 63/76] use new column name --- .../projects/projectsListView/ProjectsListViewFiltersConf.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js index 40767fad9c..3a15fda6c7 100644 --- a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js +++ b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js @@ -417,7 +417,7 @@ export const PROJECT_LIST_VIEW_FILTERS_CONFIG = { ], }, { - name: "self_and_children_council_districts", + name: "project_and_child_project_council_districts", label: "Council Districts", placeholder: "District", type: "array", From 18c48462e9ae5c6b9c9590ab53ef756e2b95ad44 Mon Sep 17 00:00:00 2001 From: John Clary Date: Tue, 13 Feb 2024 17:36:57 -0500 Subject: [PATCH 64/76] make description optional in update mutation --- moped-editor/src/queries/components.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-editor/src/queries/components.js b/moped-editor/src/queries/components.js index 52b846ad6d..9bd383827e 100644 --- a/moped-editor/src/queries/components.js +++ b/moped-editor/src/queries/components.js @@ -207,7 +207,7 @@ export const UPDATE_COMPONENT_ATTRIBUTES = gql` mutation UpdateSignalComponent( $projectComponentId: Int! $componentId: Int! - $description: String! + $description: String $subcomponents: [moped_proj_components_subcomponents_insert_input!]! $workTypes: [moped_proj_component_work_types_insert_input!]! $signalsToCreate: [feature_signals_insert_input!]! From 905acb0c31d365a88728a466da107695e707f76f Mon Sep 17 00:00:00 2001 From: John Clary Date: Tue, 13 Feb 2024 17:37:37 -0500 Subject: [PATCH 65/76] cast empty description to null and log errors as errors --- .../projectView/KnackWorkOrderTable.js | 87 +++++++++++++++++++ .../ProjectComponents/EditAttributesModal.js | 4 +- 2 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 moped-editor/src/views/projects/projectView/KnackWorkOrderTable.js diff --git a/moped-editor/src/views/projects/projectView/KnackWorkOrderTable.js b/moped-editor/src/views/projects/projectView/KnackWorkOrderTable.js new file mode 100644 index 0000000000..78fc6b5f2d --- /dev/null +++ b/moped-editor/src/views/projects/projectView/KnackWorkOrderTable.js @@ -0,0 +1,87 @@ +import { useMemo, useState, useCallback, useEffect } from "react"; +import { useParams } from "react-router-dom"; +import { useQuery, useMutation } from "@apollo/client"; +import Box from "@mui/material/Box"; +import CircularProgress from "@mui/material/CircularProgress"; +import { DataGrid } from "@mui/x-data-grid"; +import EditOutlinedIcon from "@mui/icons-material/EditOutlined"; +import DeleteOutlineIcon from "@mui/icons-material/DeleteOutline"; +import IconButton from "@mui/material/IconButton"; +import Link from "@mui/material/Link"; +import ApolloErrorHandler from "src/components/ApolloErrorHandler"; +// import WorkActivityToolbar from "./ProjectWorkActivityToolbar"; +// import ProjectWorkActivitiesDialog from "./ProjectWorkActivityDialog"; +import { getUserFullName } from "src/utils/userNames"; + +import { currencyFormatter } from "src/utils/numberFormatters"; + +// https://builder.knack.com/atd/test-25-feb-2022-dt-austin-transportation-data-tracker/pages/scene_514/views/view_3108/table + +const HEADERS = { + "X-Knack-Application-Id": "621958d0b5ab96001edcb4b1", + "X-Knack-REST-API-KEY": "knack", +}; +const SCENE = "scene_514"; +const VIEW = "view_3108"; +const FILTERS = { + match: "and", + rules: [ + { + field: "field_3965", + operator: "is", + value: "62195e71f538d8072b168ac0", + }, + ], +}; + +const URL = `https://api.knack.com/v1/pages/${SCENE}/views/${VIEW}/records?filters=${encodeURIComponent( + JSON.stringify(FILTERS) +)}`; + +export const useKnackData = (url, headers) => { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + useEffect(() => { + fetch(url, { headers }) + .then((response) => response.json()) + .then( + (json) => { + setData(json); + }, + (error) => { + setError(error.toString()); + } + ); + }, [url]); + error && console.error(error); + return { data, error, loading: !data && !error }; +}; + +const KnackWorkOrderTable = ({ projectId, knackProjectId }) => { + const { error, loading, data } = useKnackData(URL, HEADERS); + if (loading || !data) return ; + + console.log("HIIIII", data); + return ( + + {/* "auto"} + hideFooterPagination={true} + localeText={{ noRowsLabel: "No work activites" }} + rows={activities} + slots={{ + toolbar: WorkActivityToolbar, + }} + slotProps={{ + toolbar: { onClick: onClickAddActivity }, + }} + /> */} + + ); +}; + +export default KnackWorkOrderTable; diff --git a/moped-editor/src/views/projects/projectView/ProjectComponents/EditAttributesModal.js b/moped-editor/src/views/projects/projectView/ProjectComponents/EditAttributesModal.js index 72dab6a089..e7c8080c00 100644 --- a/moped-editor/src/views/projects/projectView/ProjectComponents/EditAttributesModal.js +++ b/moped-editor/src/views/projects/projectView/ProjectComponents/EditAttributesModal.js @@ -108,7 +108,7 @@ const EditAttributesModal = ({ ); }) .catch((error) => { - console.log(error); + console.error(error); }); }; @@ -122,7 +122,7 @@ const EditAttributesModal = ({ description: clickedComponent.description?.length > 0 ? clickedComponent.description - : "", + : null, subcomponents: makeSubcomponentsFormFieldValues( clickedComponent.moped_proj_components_subcomponents ), From e0e373b9c7b0cc3a1be4be515ca2d97518f44bc9 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 13 Feb 2024 16:38:50 -0600 Subject: [PATCH 66/76] adjust operator names based on feedback --- .../src/components/GridTable/FiltersCommonOperators.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moped-editor/src/components/GridTable/FiltersCommonOperators.js b/moped-editor/src/components/GridTable/FiltersCommonOperators.js index 9e8e0ce4f2..0f5ede3767 100644 --- a/moped-editor/src/components/GridTable/FiltersCommonOperators.js +++ b/moped-editor/src/components/GridTable/FiltersCommonOperators.js @@ -170,28 +170,28 @@ export const FiltersCommonOperators = { }, council_districts_array_is_null: { operator: "_is_null", - label: "No", + label: "is blank", description: "Project has no mapped components in council districts", type: "array", envelope: "true", }, council_districts_array_is_not_null: { operator: "_is_null", - label: "Yes", + label: "is not blank", description: "Project has components in council districts", type: "array", envelope: "false", }, council_districts_array_contains: { operator: "_contains", - label: "Contains", + label: "contains", description: "Project has components in a given council district", type: "array", envelope: false, }, council_districts_array_is: { operator: "_eq", - label: "Is", + label: "is", description: "Project has components in, and only in, specific council districts", type: "array", envelope: false, From cb62a6f953727f5712659a317159aa74064f2e98 Mon Sep 17 00:00:00 2001 From: John Clary Date: Tue, 13 Feb 2024 17:39:11 -0500 Subject: [PATCH 67/76] rename the query --- moped-editor/src/queries/components.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-editor/src/queries/components.js b/moped-editor/src/queries/components.js index 9bd383827e..1377db478d 100644 --- a/moped-editor/src/queries/components.js +++ b/moped-editor/src/queries/components.js @@ -204,7 +204,7 @@ export const GET_PROJECT_COMPONENTS = gql` // are switched to is_deleted = false by the mutation). It also handles deleting/creating of signal // features and intersection/point features as needed when component type is changed export const UPDATE_COMPONENT_ATTRIBUTES = gql` - mutation UpdateSignalComponent( + mutation UpdateProjectComponent( $projectComponentId: Int! $componentId: Int! $description: String From a53e9c9b3e7f83f7c61fb2089d6165346e1ec51d Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 13 Feb 2024 16:43:10 -0600 Subject: [PATCH 68/76] rework my backwards conditionals; H/T John --- .../useProjectListViewQuery/useAdvancedSearch.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch.js b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch.js index cb382cc73e..236fa19d76 100644 --- a/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch.js +++ b/moped-editor/src/views/projects/projectsListView/useProjectListViewQuery/useAdvancedSearch.js @@ -73,15 +73,9 @@ const makeAdvancedSearchWhereFilters = (filters) => // If it is a number or boolean, it does not need quotation marks // Otherwise, add quotation marks for the query to identify as string - if (["number", "boolean"].includes(type)) { - // eslint-disable-next-line no-self-assign - value = value; - } else if (type === "array") { - // a virtue of this stringification of an array, we actually support - // comma delimited lists of districts and `graphql-engine` will do the - // correct set algebra + if (type === "array") { value = `[${value}]`; - } else { + } else if (!["number", "boolean"].includes(type)) { value = `"${value}"`; } } else { From d192ba173cd17774d06a700de3e84c0dc60d5066 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 13 Feb 2024 16:47:51 -0600 Subject: [PATCH 69/76] use existing validation on council district input --- moped-editor/src/components/GridTable/helpers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moped-editor/src/components/GridTable/helpers.js b/moped-editor/src/components/GridTable/helpers.js index a6dfa34c91..06d02f6bc7 100644 --- a/moped-editor/src/components/GridTable/helpers.js +++ b/moped-editor/src/components/GridTable/helpers.js @@ -31,7 +31,8 @@ export const generateEmptyFilter = () => { */ export const checkIsValidInput = (filterParameter, type) => { // If we are testing a number type field with a non null value - if (type === "number" && !!filterParameter.value) { + // if (type === "number" && !!filterParameter.value) { + if (["number", "array"].includes(type) && !!filterParameter.value) { // Return whether string only contains digits return !/[^0-9]/.test(filterParameter.value); } From 304df1f42f1f5e655f83048e4e42c081be3706f7 Mon Sep 17 00:00:00 2001 From: John Clary Date: Tue, 13 Feb 2024 17:51:17 -0500 Subject: [PATCH 70/76] delete code i didn't want to commit --- .../projectView/KnackWorkOrderTable.js | 87 ------------------- 1 file changed, 87 deletions(-) delete mode 100644 moped-editor/src/views/projects/projectView/KnackWorkOrderTable.js diff --git a/moped-editor/src/views/projects/projectView/KnackWorkOrderTable.js b/moped-editor/src/views/projects/projectView/KnackWorkOrderTable.js deleted file mode 100644 index 78fc6b5f2d..0000000000 --- a/moped-editor/src/views/projects/projectView/KnackWorkOrderTable.js +++ /dev/null @@ -1,87 +0,0 @@ -import { useMemo, useState, useCallback, useEffect } from "react"; -import { useParams } from "react-router-dom"; -import { useQuery, useMutation } from "@apollo/client"; -import Box from "@mui/material/Box"; -import CircularProgress from "@mui/material/CircularProgress"; -import { DataGrid } from "@mui/x-data-grid"; -import EditOutlinedIcon from "@mui/icons-material/EditOutlined"; -import DeleteOutlineIcon from "@mui/icons-material/DeleteOutline"; -import IconButton from "@mui/material/IconButton"; -import Link from "@mui/material/Link"; -import ApolloErrorHandler from "src/components/ApolloErrorHandler"; -// import WorkActivityToolbar from "./ProjectWorkActivityToolbar"; -// import ProjectWorkActivitiesDialog from "./ProjectWorkActivityDialog"; -import { getUserFullName } from "src/utils/userNames"; - -import { currencyFormatter } from "src/utils/numberFormatters"; - -// https://builder.knack.com/atd/test-25-feb-2022-dt-austin-transportation-data-tracker/pages/scene_514/views/view_3108/table - -const HEADERS = { - "X-Knack-Application-Id": "621958d0b5ab96001edcb4b1", - "X-Knack-REST-API-KEY": "knack", -}; -const SCENE = "scene_514"; -const VIEW = "view_3108"; -const FILTERS = { - match: "and", - rules: [ - { - field: "field_3965", - operator: "is", - value: "62195e71f538d8072b168ac0", - }, - ], -}; - -const URL = `https://api.knack.com/v1/pages/${SCENE}/views/${VIEW}/records?filters=${encodeURIComponent( - JSON.stringify(FILTERS) -)}`; - -export const useKnackData = (url, headers) => { - const [data, setData] = useState(null); - const [error, setError] = useState(null); - useEffect(() => { - fetch(url, { headers }) - .then((response) => response.json()) - .then( - (json) => { - setData(json); - }, - (error) => { - setError(error.toString()); - } - ); - }, [url]); - error && console.error(error); - return { data, error, loading: !data && !error }; -}; - -const KnackWorkOrderTable = ({ projectId, knackProjectId }) => { - const { error, loading, data } = useKnackData(URL, HEADERS); - if (loading || !data) return ; - - console.log("HIIIII", data); - return ( - - {/* "auto"} - hideFooterPagination={true} - localeText={{ noRowsLabel: "No work activites" }} - rows={activities} - slots={{ - toolbar: WorkActivityToolbar, - }} - slotProps={{ - toolbar: { onClick: onClickAddActivity }, - }} - /> */} - - ); -}; - -export default KnackWorkOrderTable; From b2c062b087b1667774529f34602f38c54eb668cf Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 14 Feb 2024 09:47:30 -0600 Subject: [PATCH 71/76] Add breadcrumbs to the latest migration that set the views --- moped-database/views/component_arcgis_online_view.sql | 2 ++ moped-database/views/project_geography.sql | 2 +- moped-database/views/project_list_view.sql | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/moped-database/views/component_arcgis_online_view.sql b/moped-database/views/component_arcgis_online_view.sql index a4180ad908..97f8084c82 100644 --- a/moped-database/views/component_arcgis_online_view.sql +++ b/moped-database/views/component_arcgis_online_view.sql @@ -1,3 +1,5 @@ +-- moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql + CREATE OR REPLACE VIEW component_arcgis_online_view AS WITH work_types AS ( SELECT diff --git a/moped-database/views/project_geography.sql b/moped-database/views/project_geography.sql index 67229b1225..0e832bb4f4 100644 --- a/moped-database/views/project_geography.sql +++ b/moped-database/views/project_geography.sql @@ -1,4 +1,4 @@ --- latest version 1700149643859_add_audit_fields_to_unified_features_view +-- moped-database/migrations/1707416196012_add_check_for_deleted_components/up.sql CREATE OR REPLACE VIEW public.project_geography AS SELECT diff --git a/moped-database/views/project_list_view.sql b/moped-database/views/project_list_view.sql index 9300a50495..8be5541d80 100644 --- a/moped-database/views/project_list_view.sql +++ b/moped-database/views/project_list_view.sql @@ -1,3 +1,5 @@ +-- moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql + CREATE OR REPLACE VIEW public.project_list_view AS WITH project_person_list_lookup AS ( SELECT From 29cce481cf85d18bf80c4f2622b13cdb77606bf5 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 14 Feb 2024 10:28:13 -0600 Subject: [PATCH 72/76] Use sentence case for labels - H/T Chia - thank you --- .../projects/projectsListView/ProjectsListViewFiltersConf.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js index 3a15fda6c7..fe302b2072 100644 --- a/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js +++ b/moped-editor/src/views/projects/projectsListView/ProjectsListViewFiltersConf.js @@ -400,7 +400,7 @@ export const PROJECT_LIST_VIEW_FILTERS_CONFIG = { }, { name: "children_project_ids", - label: "Has Subprojects", + label: "Has subprojects", placeholder: "Subproject", type: "array", operators: ["subprojects_array_is_null", "subprojects_array_is_not_null"], @@ -418,7 +418,7 @@ export const PROJECT_LIST_VIEW_FILTERS_CONFIG = { }, { name: "project_and_child_project_council_districts", - label: "Council Districts", + label: "Council districts", placeholder: "District", type: "array", operators: [ From 4efcd25fbacd314ea6063f9011c7b6a6af9e03b7 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 14 Feb 2024 14:40:07 -0600 Subject: [PATCH 73/76] Deleted features of not-deleted components must be excluded as well! --- .../1707416196011_add_council_districts_to_project_view/up.sql | 3 ++- moped-database/views/project_list_view.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql index bc8bc410fd..cee3df850f 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql @@ -98,7 +98,8 @@ project_district_association AS ( LEFT JOIN moped_proj_components ON (moped_project.project_id = moped_proj_components.project_id) LEFT JOIN features ON (moped_proj_components.project_component_id = features.component_id) LEFT JOIN features_council_districts ON features.id = features_council_districts.feature_id - WHERE moped_proj_components.is_deleted IS false + WHERE features.is_deleted IS false + AND moped_proj_components.is_deleted IS false ), parent_child_project_map AS ( diff --git a/moped-database/views/project_list_view.sql b/moped-database/views/project_list_view.sql index 8be5541d80..a1ff551621 100644 --- a/moped-database/views/project_list_view.sql +++ b/moped-database/views/project_list_view.sql @@ -97,7 +97,8 @@ project_district_association AS ( LEFT JOIN moped_proj_components ON (moped_project.project_id = moped_proj_components.project_id) LEFT JOIN features ON (moped_proj_components.project_component_id = features.component_id) LEFT JOIN features_council_districts ON features.id = features_council_districts.feature_id - WHERE moped_proj_components.is_deleted IS false + WHERE features.is_deleted IS false + AND moped_proj_components.is_deleted IS false ), parent_child_project_map AS ( From 7963b9509b3ceafd9d7fac39c5f2aa7f396bf4aa Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 14 Feb 2024 17:38:26 -0600 Subject: [PATCH 74/76] restore to querying both alone and alone + children data --- .../up.sql | 12 ++++++------ moped-database/views/project_list_view.sql | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql index cee3df850f..6bcc727835 100644 --- a/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql +++ b/moped-database/migrations/1707416196011_add_council_districts_to_project_view/up.sql @@ -117,16 +117,16 @@ project_district_association AS ( SELECT projects.project_id, - array_agg(DISTINCT districts.council_district_id) FILTER ( - WHERE districts.council_district_id IS NOT null + array_agg(DISTINCT project_districts.council_district_id) FILTER ( + WHERE project_districts.council_district_id IS NOT null ) AS project_council_districts, - array_agg(DISTINCT districts.council_district_id) FILTER ( - WHERE districts.council_district_id IS NOT null + array_agg(DISTINCT project_and_children_districts.council_district_id) FILTER ( + WHERE project_and_children_districts.council_district_id IS NOT null ) AS project_and_child_project_council_districts FROM parent_child_project_map AS projects LEFT JOIN - project_council_district_map AS districts - ON (projects.self_and_children_project_ids = districts.project_id) + project_council_district_map AS project_and_children_districts + ON (projects.self_and_children_project_ids = project_and_children_districts.project_id) LEFT JOIN project_council_district_map AS project_districts ON (projects.project_id = project_districts.project_id) GROUP BY projects.project_id ) diff --git a/moped-database/views/project_list_view.sql b/moped-database/views/project_list_view.sql index a1ff551621..abdb28c2a9 100644 --- a/moped-database/views/project_list_view.sql +++ b/moped-database/views/project_list_view.sql @@ -116,16 +116,16 @@ project_district_association AS ( SELECT projects.project_id, - array_agg(DISTINCT districts.council_district_id) FILTER ( - WHERE districts.council_district_id IS NOT null + array_agg(DISTINCT project_districts.council_district_id) FILTER ( + WHERE project_districts.council_district_id IS NOT null ) AS project_council_districts, - array_agg(DISTINCT districts.council_district_id) FILTER ( - WHERE districts.council_district_id IS NOT null + array_agg(DISTINCT project_and_children_districts.council_district_id) FILTER ( + WHERE project_and_children_districts.council_district_id IS NOT null ) AS project_and_child_project_council_districts FROM parent_child_project_map AS projects LEFT JOIN - project_council_district_map AS districts - ON (projects.self_and_children_project_ids = districts.project_id) + project_council_district_map AS project_and_children_districts + ON (projects.self_and_children_project_ids = project_and_children_districts.project_id) LEFT JOIN project_council_district_map AS project_districts ON (projects.project_id = project_districts.project_id) GROUP BY projects.project_id ) From edc111ff7dc880f71648c3cfe644dff8857f5011 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 15 Feb 2024 10:33:09 -0600 Subject: [PATCH 75/76] clean up commented code, H/T Mike --- moped-editor/src/components/GridTable/helpers.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moped-editor/src/components/GridTable/helpers.js b/moped-editor/src/components/GridTable/helpers.js index 06d02f6bc7..0e26c0d79f 100644 --- a/moped-editor/src/components/GridTable/helpers.js +++ b/moped-editor/src/components/GridTable/helpers.js @@ -30,8 +30,7 @@ export const generateEmptyFilter = () => { * @returns {boolean} */ export const checkIsValidInput = (filterParameter, type) => { - // If we are testing a number type field with a non null value - // if (type === "number" && !!filterParameter.value) { + // If we are testing a number or array type field with a non null value if (["number", "array"].includes(type) && !!filterParameter.value) { // Return whether string only contains digits return !/[^0-9]/.test(filterParameter.value); From 4a9b360edd3b950a6bb9545bcb7796caf49aaaef Mon Sep 17 00:00:00 2001 From: John Clary Date: Thu, 15 Feb 2024 14:46:38 -0500 Subject: [PATCH 76/76] exclude updated_at from trigger column set --- moped-database/metadata/tables.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/moped-database/metadata/tables.yaml b/moped-database/metadata/tables.yaml index 5e58dd7513..b1d68ed6c1 100644 --- a/moped-database/metadata/tables.yaml +++ b/moped-database/metadata/tables.yaml @@ -3469,7 +3469,21 @@ insert: columns: '*' update: - columns: '*' + columns: + - is_phase_start_confirmed + - is_phase_end_confirmed + - created_at + - phase_start + - phase_end + - is_deleted + - project_id + - created_by_user_id + - updated_by_user_id + - subphase_id + - phase_description + - is_current_phase + - project_phase_id + - phase_id retry_conf: interval_sec: 10 num_retries: 0