Adds in dynamic publishers component. Allows filter by publishers. us…

…eQueryParams now only uses Array type... again.

orp/config/urls.py

@@ -100,7 +100,7 @@ def rebuild_cache(self, request, *args, **kwargs):
         tx_begin = time.time()
         try:
             clear_all_documents()
-            config = SearchDocumentConfig(search_query="", timeout=10)
+            config = SearchDocumentConfig(search_query="", timeout=20)
             Legislation().build_cache(config)
             PublicGateway().build_cache(config)
         except Exception as e:
@@ -130,7 +130,10 @@ def publishers(self, request, *args, **kwargs):
             publishers = get_publisher_names()
 
             results = [
-                {"name": item["publisher"], "key": item["publisher_id"]}
+                {
+                    "label": item["trimmed_publisher"],
+                    "name": item["trimmed_publisher_id"],
+                }
                 for item in publishers
             ]
 

orp/orp_search/utils/search.py

@@ -9,7 +9,7 @@
 from orp_search.utils.terms import sanitize_input
 
 from django.contrib.postgres.search import SearchQuery, SearchVector
-from django.db.models import Q, QuerySet
+from django.db.models import F, Func, Q, QuerySet
 from django.http import HttpRequest
 
 logger = logging.getLogger(__name__)
@@ -189,15 +189,27 @@ def search(context: dict, request: HttpRequest) -> dict:
     return context
 
 
+class Trim(Func):
+    function = "TRIM"
+    template = "%(function)s(%(expressions)s)"
+
+
 def get_publisher_names():
     logger.info("getting publisher names...")
     publishers_list = []
 
     try:
-        publishers_list = DataResponseModel.objects.values(
-            "publisher",
-            "publisher_id",
-        ).distinct()
+        publishers_list = (
+            DataResponseModel.objects.annotate(
+                trimmed_publisher=Trim(F("publisher")),
+                trimmed_publisher_id=Trim(F("publisher_id")),
+            )
+            .values(
+                "trimmed_publisher",
+                "trimmed_publisher_id",
+            )
+            .distinct()
+        )
 
     except Exception as e:
         logger.error(f"error getting publisher names: {e}")

react_front_end/src/App.js

@@ -1,7 +1,7 @@
 import { useState, useEffect, useMemo, useCallback } from "react"
 import { useQueryParams } from "./hooks/useQueryParams"
 import { fetchData } from "./utils/fetch-drf"
-import { DOCUMENT_TYPES, PUBLISHERS } from "./utils/constants"
+import { DOCUMENT_TYPES, PUBLISHERS_URL } from "./utils/constants"
 
 import { Search } from "./components/Search"
 import { CheckboxFilter } from "./components/CheckboxFilter"
@@ -12,30 +12,48 @@ import { ResultsCount } from "./components/ResultsCount"
 import { SortSelect } from "./components/SortSelect"
 import { NoResultsContent } from "./components/NoResultsContent"
 
-const generateCheckedState = (checkboxes, queryValues) => checkboxes.map(({ name }) => queryValues.includes(name))
+const generateCheckedState = (checkboxes, queryValues) => {
+  return checkboxes.map(({ name }) => queryValues.includes(name))
+}
 
 function App() {
-  const [searchQuery, setSearchQuery] = useQueryParams("search", "")
+  const [searchQuery, setSearchQuery] = useQueryParams("search", [])
   const [docTypeQuery, setDocTypeQuery] = useQueryParams("document_type", [])
   const [publisherQuery, setPublisherQuery] = useQueryParams("publisher", [])
-  const [sortQuery, setSortQuery] = useQueryParams("sort", "recent")
-  const [pageQuery, setPageQuery] = useQueryParams("page", 1)
+  const [sortQuery, setSortQuery] = useQueryParams("sort", ["recent"])
+  const [pageQuery, setPageQuery] = useQueryParams("page", [1])
 
-  const [searchInput, setSearchInput] = useState(searchQuery) // Set initial state to query parameter value
+  const [searchInput, setSearchInput] = useState(searchQuery[0]) // Set initial state to query parameter value
   const [data, setData] = useState([])
   const [isLoading, setIsLoading] = useState(true)
   const [isSearchSubmitted, setIsSearchSubmitted] = useState(false)
+  const [publishers, setPublishers] = useState([]) // Add publishers state
+  const [publisherCheckedState, setPublisherCheckedState] = useState([])
+
+  useEffect(() => {
+    const fetchPublishers = async () => {
+      try {
+        const response = await fetch(PUBLISHERS_URL)
+        if (!response.ok) {
+          throw new Error("Network response was not ok")
+        }
+        const data = await response.json()
+        setPublishers(data.results)
+        setPublisherCheckedState(generateCheckedState(data.results, publisherQuery))
+      } catch (error) {
+        console.error("There was a problem with fetching the publishers:", error)
+      }
+    }
+
+    fetchPublishers()
+  }, [])
 
   // Memoize the initial checked state for document types and publishers
   const initialDocumentTypeCheckedState = useMemo(
     () => generateCheckedState(DOCUMENT_TYPES, docTypeQuery),
     [docTypeQuery],
   )
-  const initialPublisherCheckedState = useMemo(() => generateCheckedState(PUBLISHERS, publisherQuery), [publisherQuery])
-
-  // Set initial checked state as array of booleans for checkboxes based on query params
   const [documentTypeCheckedState, setDocumentTypeCheckedState] = useState(initialDocumentTypeCheckedState)
-  const [publisherCheckedState, setPublisherCheckedState] = useState(initialPublisherCheckedState)
 
   // Memoize the handleSearchChange function
   const handleSearchChange = useCallback(
@@ -55,7 +73,7 @@ function App() {
     if (filterName === "docType") {
       updateQueryAndState(docTypeQuery, setDocTypeQuery, setDocumentTypeCheckedState, DOCUMENT_TYPES)
     } else if (filterName === "publisher") {
-      updateQueryAndState(publisherQuery, setPublisherQuery, setPublisherCheckedState, PUBLISHERS)
+      updateQueryAndState(publisherQuery, setPublisherQuery, setPublisherCheckedState, publishers)
     }
   }
 
@@ -64,7 +82,7 @@ function App() {
     setDocTypeQuery([])
     setPublisherQuery([])
     setDocumentTypeCheckedState(generateCheckedState(DOCUMENT_TYPES, []))
-    setPublisherCheckedState(generateCheckedState(PUBLISHERS, []))
+    setPublisherCheckedState(generateCheckedState(publishers, []))
   }
 
   const fetchDataWithLoading = async (queryString) => {
@@ -81,15 +99,15 @@ function App() {
 
   const handleSearchSubmit = useCallback(() => {
     setIsSearchSubmitted(true)
-    setSearchQuery(searchInput)
-    setPageQuery(1) // Set the page to 1 when a new search is made
+    setSearchQuery([searchInput])
+    setPageQuery([1]) // Set the page to 1 when a new search is made
 
     const filterParams = {
       ...(searchInput.length > 0 && { search: searchInput }),
       ...(docTypeQuery.length > 0 && { document_type: docTypeQuery }),
       ...(publisherQuery.length > 0 && { publisher: publisherQuery }),
-      sort: sortQuery,
-      page: 1, // Set page to 1 for new search
+      sort: sortQuery.join(","),
+      page: [1], // Set page to 1 for new search
     }
 
     fetchDataWithLoading(filterParams)
@@ -103,11 +121,11 @@ function App() {
 
     const handler = setTimeout(() => {
       const filterParams = {
-        ...(searchQuery.length > 0 && { search: searchQuery }),
+        ...(searchQuery.length > 0 && { search: searchQuery.join(",") }),
         ...(docTypeQuery.length > 0 && { document_type: docTypeQuery }),
         ...(publisherQuery.length > 0 && { publisher: publisherQuery }),
-        sort: sortQuery,
-        page: pageQuery,
+        sort: sortQuery.join(","),
+        page: pageQuery.join(","),
       }
 
       fetchDataWithLoading(filterParams)
@@ -146,26 +164,30 @@ function App() {
             <legend className="govuk-fieldset__legend govuk-fieldset__legend--m">
               <h2 className="govuk-fieldset__heading">Published by</h2>
             </legend>
-            <CheckboxFilter
-              checkboxData={PUBLISHERS}
-              checkedState={publisherCheckedState}
-              setCheckedState={setPublisherCheckedState}
-              setQueryParams={setPublisherQuery}
-              withSearch={true}
-              setIsLoading={setIsLoading}
-            />
+            {publishers ? (
+              <CheckboxFilter
+                checkboxData={publishers}
+                checkedState={publisherCheckedState}
+                setCheckedState={setPublisherCheckedState}
+                setQueryParams={setPublisherQuery}
+                withSearch={true}
+                setIsLoading={setIsLoading}
+              />
+            ) : (
+              <p>Loading publishers...</p>
+            )}
           </fieldset>
         </div>
         <hr className="govuk-section-break govuk-section-break--m govuk-section-break--visible" />
         <p className="govuk-body">
           <a
             id="download-csv-link"
             href={`download_csv/?${new URLSearchParams({
-              search: searchQuery,
+              search: searchQuery.join(","),
               document_type: docTypeQuery.join(","),
               publisher: publisherQuery.join(","),
-              sort: sortQuery,
-              page: pageQuery,
+              sort: sortQuery.join(","),
+              page: pageQuery.join(","),
             }).toString()}`}
             className="govuk-link govuk-link--no-visited-state govuk-!-float-right"
           >
@@ -197,19 +219,20 @@ function App() {
             documentTypeCheckedState={documentTypeCheckedState}
             publisherCheckedState={publisherCheckedState}
             removeFilter={handleDeleteFilter}
+            publishers={publishers}
           />
         ) : (
           <hr className="govuk-section-break govuk-section-break--m govuk-section-break--visible" />
         )}
-        <SortSelect sortQuery={sortQuery} setSortQuery={setSortQuery} />
+        <SortSelect sortQuery={sortQuery[0]} setSortQuery={setSortQuery} />
         <hr className="govuk-section-break govuk-section-break--m govuk-section-break--visible" />
         {data.results_total_count === 0 && !isLoading ? (
           <NoResultsContent />
         ) : (
-          <Results results={data.results} isLoading={isLoading} searchQuery={searchQuery} />
+          <Results results={data.results} isLoading={isLoading} searchQuery={searchQuery[0]} />
         )}
 
-        <Pagination pageData={data} pageQuery={pageQuery} setPageQuery={setPageQuery} />
+        <Pagination pageData={data} pageQuery={pageQuery[0]} setPageQuery={setPageQuery} />
       </div>
     </div>
   )

react_front_end/src/components/AppliedFilters.js

@@ -1,13 +1,13 @@
-import { DOCUMENT_TYPES, PUBLISHERS } from "../utils/constants"
+import { DOCUMENT_TYPES } from "../utils/constants"
 
-function AppliedFilters({ documentTypeCheckedState, publisherCheckedState, removeFilter }) {
+function AppliedFilters({ documentTypeCheckedState, publisherCheckedState, removeFilter, publishers }) {
   // Combine checked document types and publishers into a single array
   const checkedFilters = [
     ...documentTypeCheckedState
       .map((item, index) => (item ? { type: "docType", ...DOCUMENT_TYPES[index] } : null))
       .filter((item) => item !== null),
     ...publisherCheckedState
-      .map((item, index) => (item ? { type: "publisher", ...PUBLISHERS[index] } : null))
+      .map((item, index) => (item ? { type: "publisher", ...publishers[index] } : null))
       .filter((item) => item !== null),
   ]
 

react_front_end/src/components/Results.js

@@ -1,9 +1,7 @@
 import { SkeletonResults } from "./SkeletonResults"
 import { formatDateToGovukStyle } from "../utils/date"
 
-function Results({ results, isLoading, searchQuery }) {
-  // console.log("Results", results)
-
+function Results({ results, isLoading, searchQuery = "" }) {
   if (isLoading) {
     return <SkeletonResults />
   }

react_front_end/src/hooks/useQueryParams.js

@@ -1,5 +1,8 @@
 import { useState } from "react"
 
+// Custom hook to get and set query parameters
+// Always expects, and returns an array of values
+
 const getQuery = () => {
   if (typeof window !== "undefined") {
     return new URLSearchParams(window.location.search)
@@ -8,13 +11,11 @@ const getQuery = () => {
 }
 
 const getQueryStringVal = (key) => {
-  const values = getQuery().getAll(key)
-  return values.length > 1 ? values : values[0] || ""
+  return getQuery().getAll(key)
 }
 
 const useQueryParams = (key, defaultVal = []) => {
-  const initialQuery = getQueryStringVal(key)
-  const [query, setQuery] = useState(initialQuery.length ? initialQuery : defaultVal)
+  const [query, setQuery] = useState(getQueryStringVal(key).length ? getQueryStringVal(key) : defaultVal)
 
   const updateUrl = (newVals) => {
     setQuery(newVals)
@@ -25,21 +26,13 @@ const useQueryParams = (key, defaultVal = []) => {
     query.delete(key)
 
     // Set new values for the key
-    if (Array.isArray(newVals)) {
-      newVals.forEach((val) => {
-        if (typeof val === "string" && val.trim() !== "") {
-          query.append(key, val)
-        } else if (typeof val === "number") {
-          query.append(key, val.toString())
-        }
-      })
-    } else {
-      if (typeof newVals === "string" && newVals.trim() !== "") {
-        query.append(key, newVals)
-      } else if (typeof newVals === "number") {
-        query.append(key, newVals.toString())
+    newVals.map((val) => {
+      if (typeof val === "string" && val.trim() !== "") {
+        query.append(key, val)
+      } else if (typeof val === "number") {
+        query.append(key, val.toString())
       }
-    }
+    })
 
     if (typeof window !== "undefined") {
       const { protocol, pathname, host } = window.location

react_front_end/src/utils/constants.js

@@ -1,6 +1,6 @@
-const DRF_API_URL = "http://localhost:8081/api/v1"
+const DRF_API_URL = "/api/v1"
 export const SEARCH_URL = `${DRF_API_URL}/search`
-// export const PUBLISHERS_URL = `${DRF_API_URL}/retrieve/publishers/`
+export const PUBLISHERS_URL = `${DRF_API_URL}/retrieve/publishers/`
 
 // These should come from the API/Django backend, but for now they are hardcoded
 export const DOCUMENT_TYPES = [
@@ -17,50 +17,3 @@ export const DOCUMENT_TYPES = [
     label: "Standards",
   },
 ]
-
-export const PUBLISHERS = [
-  {
-    name: "healthandsafetyexecutive",
-    label: "Health and Safety Executive",
-  },
-  {
-    name: "civilaviationauthority",
-    label: "Civil Aviation Authority",
-  },
-  {
-    name: "environmentagency",
-    label: "Environment Agency",
-  },
-  {
-    name: "defra",
-    label: "Defra",
-  },
-  {
-    name: "officeofgasandelectricitymarkets",
-    label: "Office of Gas and Electricity Markets",
-  },
-  {
-    name: "officeofrailandroad",
-    label: "Office of Rail and Road",
-  },
-  {
-    name: "naturalengland",
-    label: "Natural England",
-  },
-  {
-    name: "historicengland",
-    label: "Historic England",
-  },
-  {
-    name: "nationalhighways",
-    label: "National Highways",
-  },
-  {
-    name: "homesengland",
-    label: "Homes England",
-  },
-  {
-    name: "departmentfortransport",
-    label: "Department for Transport",
-  },
-]