From e8d483e436a6d3514ce3361472601fccff5b31d8 Mon Sep 17 00:00:00 2001 From: lublagg Date: Fri, 9 Feb 2024 17:23:56 -0500 Subject: [PATCH 1/4] Checkpoint. --- src/components/attribute-options.tsx | 2 +- src/components/options.tsx | 10 +- src/constants/constants.ts | 2 +- src/constants/types.ts | 4 +- src/scripts/api.ts | 268 ++++++++++++++++----------- 5 files changed, 172 insertions(+), 114 deletions(-) diff --git a/src/components/attribute-options.tsx b/src/components/attribute-options.tsx index e6bb686..352d7a9 100644 --- a/src/components/attribute-options.tsx +++ b/src/components/attribute-options.tsx @@ -45,7 +45,7 @@ export const AttributeOptions: React.FC = (props) => {
= (props) => { } // If user selects a crop and no unit is selected, auto-select Yield if (optionKey === "crops" && !selectedOptions.cropUnits) { - newSelection.cropUnits = "Yield"; + newSelection.cropUnits = ["Yield"]; } // If user selects a state, de-select "All States" if (optionKey === "states" && newArray.includes("All States")) { @@ -68,7 +68,7 @@ export const Options: React.FC = (props) => { // Otherwise deselect crop units, too newSelection.crops = newArray.filter((o) => o !== e.target.value); if (newSelection.crops.length === 0) { - newSelection.cropUnits = ""; + newSelection.cropUnits = []; } } else { newSelection[optionKey] = newArray.filter((o) => o !== e.target.value); @@ -76,8 +76,8 @@ export const Options: React.FC = (props) => { } } handleSetSelectedOptions(newSelection); - } else if (optionKey === "geographicLevel" || optionKey === "cropUnits") { - handleSetSelectedOptions({[optionKey]: e.target.value}); + } else if (optionKey === "geographicLevel") { + handleSetSelectedOptions({geographicLevel: e.target.value as GeographicLevel}); } }; diff --git a/src/constants/constants.ts b/src/constants/constants.ts index a0cd53e..8b08b6d 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -128,7 +128,7 @@ export const defaultSelectedOptions: IStateOptions = { farmerDemographics: [], farmDemographics: [], economicsAndWages: [], - cropUnits: "", + cropUnits: [], crops: [], years: [] }; diff --git a/src/constants/types.ts b/src/constants/types.ts index cbfb814..8b1ed93 100644 --- a/src/constants/types.ts +++ b/src/constants/types.ts @@ -6,7 +6,7 @@ export interface IStateOptions { farmerDemographics: string[], farmDemographics: string[], economicsAndWages: string[], - cropUnits: string, + cropUnits: string[], crops: string[] years: string[] } @@ -106,4 +106,4 @@ export interface IReqCount { renameable?: boolean; deleteable?: boolean; hidden?: boolean; -} \ No newline at end of file +} diff --git a/src/scripts/api.ts b/src/scripts/api.ts index c8f6cef..bba429e 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -51,11 +51,11 @@ export const createRequest = ({attribute, geographicLevel, years, states, cropUn } = queryParams; const item = cropUnits ? - (queryParams?.short_desc as ICropDataItem)[cropUnits] : - short_desc as string[]; - const cat = cropUnits ? - (queryParams?.statisticcat_desc as ICropCategory)[cropUnits] : - statisticcat_desc; + (queryParams?.short_desc as ICropDataItem)[cropUnits] : + short_desc as string[]; +const cat = cropUnits ? + (queryParams?.statisticcat_desc as ICropCategory)[cropUnits] : + statisticcat_desc; let req = ""; if (attribute === "Total Farmers" && !years.includes("2017")) { @@ -135,11 +135,13 @@ export const getAllAttrs = (selectedOptions: IStateOptions) => { const codapColumnUnit = attrToCODAPColumnName[desc].unitInCodapTable; allAttrs.push({"name": codapColumnName, "unit": codapColumnUnit}); } - } else if (typeof short_desc === "object" && cropUnits) { - const attr = short_desc[cropUnits as keyof ICropDataItem][0]; - const codapColumnName = attrToCODAPColumnName[attr].attributeNameInCodapTable; - const codapColumnUnit = attrToCODAPColumnName[attr].unitInCodapTable; - allAttrs.push({"name": codapColumnName, "unit": codapColumnUnit}); + } else if (typeof short_desc === "object" && cropUnits.length) { + cropUnits.forEach((cropUnit) => { + const attr = short_desc[cropUnit as keyof ICropDataItem][0]; + const codapColumnName = attrToCODAPColumnName[attr].attributeNameInCodapTable; + const codapColumnUnit = attrToCODAPColumnName[attr].unitInCodapTable; + allAttrs.push({"name": codapColumnName, "unit": codapColumnUnit}); + }); } } } @@ -208,7 +210,6 @@ export const createTableFromSelections = async (selectedOptions: IStateOptions, return "success"; } else { const attrDefinitions = allAttrs.map((attr) => makeCODAPAttributeDef(attr, geographicLevel)); - console.log("attrDefinitions", attrDefinitions); await createParentCollection(dataSetName, "Data", attrDefinitions); await createItems(dataSetName, items); await createTable(dataSetName, dataSetName); @@ -221,6 +222,149 @@ export const createTableFromSelections = async (selectedOptions: IStateOptions, } }; +interface IPrepareQueryAndFetchData { + isMultiStateRegion: boolean, + years: string[], + geographicLevel: GeographicLevel, + queryParams: any, + attribute: string, + stateArray: string[], + cropUnit?: string, + selectedOptions: IStateOptions, + setReqCount: ISetReqCount +} + +const prepareQueryAndFetchData = async (props: IPrepareQueryAndFetchData) => { + const {isMultiStateRegion, years, geographicLevel, queryParams, stateArray, attribute, cropUnit, selectedOptions, setReqCount} = props; + const geoLevel = isMultiStateRegion ? "REGION : MULTI-STATE" : geographicLevel; + const yearsAvailable = years.filter(year => queryParams?.years[geographicLevel].includes(year)); + if (yearsAvailable.length) { + const params: IGetAttrDataParams = {attribute, geographicLevel: geoLevel, years: yearsAvailable, states: stateArray}; + if (cropOptions.options.includes(attribute) && cropUnit) { + params.cropUnits = cropUnit as keyof ICropDataItem; + } + const data = await getAttrData(params, selectedOptions, setReqCount); + return data; + } +}; + +interface IFindMatchingData { + isMultiStateRegion: boolean, + data: any, + item: any, + geoLevel: GeographicLevel + +} + +const findMatchingData = (props: IFindMatchingData) => { + const {isMultiStateRegion, data, item, geoLevel} = props; + const matchingData = data.filter((dataObj: any) => { + if (isMultiStateRegion) { + const {region_desc, year} = dataObj; + const regionThatIncludesState = multiRegions.find((r) => r.States.includes(item.State)); + return regionThatIncludesState?.Region.toLowerCase() === region_desc.toLowerCase() && year === Number(item.Year); + } else { + const {state_name, year} = dataObj; + const lowerItemState = item.State.toLowerCase(); + const lowerDataState = state_name.toLowerCase(); + let isSameGeoLevel; + if (geoLevel === "County") { + const lowerItemCounty = item.County.toLowerCase(); + const lowerDataCounty = dataObj.county_name.toLowerCase(); + isSameGeoLevel = lowerItemCounty === lowerDataCounty && lowerItemState === lowerDataState; + } else { + isSameGeoLevel = lowerItemState === lowerDataState; + } + return isSameGeoLevel && Number(item.Year) === year; + } + }); + return matchingData; +}; + +interface IProcessAttributeData { + attribute: string, + items: any, + geographicLevel: GeographicLevel, + years: string[], + cropUnit: string, + selectedOptions: IStateOptions, + setReqCount: ISetReqCount, + stateArray: string[] +} + +const processAttributeData = async (props: IProcessAttributeData) => { + const {attribute, items, geographicLevel, years, cropUnit, selectedOptions, setReqCount, stateArray} = props; + const queryParams = getQueryParams(attribute); + const isMultiStateRegion = queryParams?.geographicAreas[0] === "REGION : MULTI-STATE"; + const data = await prepareQueryAndFetchData({isMultiStateRegion, years, geographicLevel, queryParams, attribute, stateArray, cropUnit, selectedOptions, setReqCount}); + + items.forEach((item: any) => { + // find all the data items that match this item's location and year + const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel}); + + if (isMultiStateRegion) { + if (item.State !== "Alaska") { + const { Value } = matchingData.length > 0 && matchingData[0]; + const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; + item[codapColumnName] = Value; + } + } else if (attribute === "Acres Operated") { + // special case for handling acres operated, where we have to sum up the values of several data items + const acreTotals: IAcreTotals = { + [strings.oneTo9Acres]: { + [strings.oneTo9Acres]: 0 + }, + [strings.tenTo49Acres]: { + [strings.tenTo49Acres]: 0 + }, + [strings.fiftyTo100Acres]: { + [strings.fiftyTo69Acres]: 0, + [strings.seventyTo99Acres]: 0 + }, + [strings.oneHundredTo500Acres]: { + [strings.oneHundredTo139Acres]: 0, + [strings.oneHundredFortyTo179Acres]: 0, + [strings.oneHundredEightyTo219Acres]: 0, + [strings.twoHundredTwentyTo259Acres]: 0, + [strings.twoHundredSixtyTo499Acres]: 0 + }, + [strings.fiveHundredTo999Acres]: { + [strings.fiveHundredTo999Acres]: 0 + }, + [strings.oneThousandTo5000Acres]: { + [strings.oneThousandTo1999Acres]: 0, + [strings.twoThousandTo4999Acres]: 0 + }, + [strings.fiveThousandOrMoreAcres]: { + [strings.fiveThousandOrMoreAcres]: 0 + } + }; + const totalKeys = Object.keys(acreTotals); + for (const total of totalKeys) { + const subTotalKeys = Object.keys(acreTotals[total]); + const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); + subTotalData.forEach((dataObj: any) => { + acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + }); + const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; + // sum up all the values of acreTotals[total] + const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); + const sum = onlyNumbers.reduce((acc, cur) => acc + cur); + item[codapColumnName] = sum; + } + } else { + matchingData.forEach((dataItem: any) => { + const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; + const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; + if (codapColumnName) { + item[codapColumnName] = dataItem.Value; + } + }); + } + }); + return items; +}; + const getItems = async (selectedOptions: IStateOptions, setReqCount: ISetReqCount) => { const {geographicLevel, states, years, cropUnits, ...subOptions} = selectedOptions; const items: any = []; @@ -245,101 +389,15 @@ const getItems = async (selectedOptions: IStateOptions, setReqCount: ISetReqCoun for (const key in subOptions) { const value = subOptions[key as keyof typeof subOptions]; - for (const attribute of value) { - const queryParams = getQueryParams(attribute); - const isMultiStateRegion = queryParams?.geographicAreas[0] === "REGION : MULTI-STATE"; - const geoLevel = isMultiStateRegion ? "REGION : MULTI-STATE" : geographicLevel; - const yearsAvailable = years.filter(year => queryParams?.years[selectedOptions.geographicLevel].includes(year)); - if (yearsAvailable.length) { - const params: IGetAttrDataParams = {attribute, geographicLevel: geoLevel, years: yearsAvailable, states: stateArray}; - if (cropOptions.options.includes(attribute) && cropUnits) { - params.cropUnits = cropUnits as keyof ICropDataItem; + if (cropUnits.length > 1) { + for (const cropUnit of cropUnits) { + for (const attribute of value) { + await processAttributeData({attribute, items, geographicLevel, years, cropUnit, selectedOptions, setReqCount, stateArray}); } - const data = await getAttrData(params, selectedOptions, setReqCount); - - items.forEach((item: any) => { - // find all the data items that match this item's location and year - const matchingData = data.filter((dataObj: any) => { - if (isMultiStateRegion) { - const {region_desc, year} = dataObj; - const regionThatIncludesState = multiRegions.find((r) => r.States.includes(item.State)); - return regionThatIncludesState?.Region.toLowerCase() === region_desc.toLowerCase() && year === Number(item.Year); - } else { - const {state_name, year} = dataObj; - const lowerItemState = item.State.toLowerCase(); - const lowerDataState = state_name.toLowerCase(); - let isSameGeoLevel; - if (geoLevel === "County") { - const lowerItemCounty = item.County.toLowerCase(); - const lowerDataCounty = dataObj.county_name.toLowerCase(); - isSameGeoLevel = lowerItemCounty === lowerDataCounty && lowerItemState === lowerDataState; - } else { - isSameGeoLevel = lowerItemState === lowerDataState; - } - return isSameGeoLevel && Number(item.Year) === year; - } - }); - - if (isMultiStateRegion) { - if (item.State !== "Alaska") { - const { Value } = matchingData.length > 0 && matchingData[0]; - const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; - item[codapColumnName] = Value; - } - } else if (attribute === "Acres Operated") { - // special case for handling acres operated, where we have to sum up the values of several data items - const acreTotals: IAcreTotals = { - [strings.oneTo9Acres]: { - [strings.oneTo9Acres]: 0 - }, - [strings.tenTo49Acres]: { - [strings.tenTo49Acres]: 0 - }, - [strings.fiftyTo100Acres]: { - [strings.fiftyTo69Acres]: 0, - [strings.seventyTo99Acres]: 0 - }, - [strings.oneHundredTo500Acres]: { - [strings.oneHundredTo139Acres]: 0, - [strings.oneHundredFortyTo179Acres]: 0, - [strings.oneHundredEightyTo219Acres]: 0, - [strings.twoHundredTwentyTo259Acres]: 0, - [strings.twoHundredSixtyTo499Acres]: 0 - }, - [strings.fiveHundredTo999Acres]: { - [strings.fiveHundredTo999Acres]: 0 - }, - [strings.oneThousandTo5000Acres]: { - [strings.oneThousandTo1999Acres]: 0, - [strings.twoThousandTo4999Acres]: 0 - }, - [strings.fiveThousandOrMoreAcres]: { - [strings.fiveThousandOrMoreAcres]: 0 - } - }; - const totalKeys = Object.keys(acreTotals); - for (const total of totalKeys) { - const subTotalKeys = Object.keys(acreTotals[total]); - const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); - subTotalData.forEach((dataObj: any) => { - acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); - }); - const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; - // sum up all the values of acreTotals[total] - const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); - const sum = onlyNumbers.reduce((acc, cur) => acc + cur); - item[codapColumnName] = sum; - } - } else { - matchingData.forEach((dataItem: any) => { - const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; - const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; - if (codapColumnName) { - item[codapColumnName] = dataItem.Value; - } - }); - } - }); + } + } else { + for (const attribute of value) { + await processAttributeData({attribute, items, geographicLevel, years, cropUnit: cropUnits[0], selectedOptions, setReqCount, stateArray}); } } } From de65ec862f2bd190169eca6e2bf6166ca9e7c746 Mon Sep 17 00:00:00 2001 From: lublagg Date: Mon, 12 Feb 2024 18:14:03 -0500 Subject: [PATCH 2/4] Return undefined for data if bad request. --- src/constants/queryHeaders.ts | 32 ++++++-- src/scripts/api.ts | 135 ++++++++++++++++++---------------- 2 files changed, 96 insertions(+), 71 deletions(-) diff --git a/src/constants/queryHeaders.ts b/src/constants/queryHeaders.ts index a0f1b2b..b92bba9 100644 --- a/src/constants/queryHeaders.ts +++ b/src/constants/queryHeaders.ts @@ -30,11 +30,7 @@ const sharedLaborHeaders = { const sharedCropHeaders = { sect_desc: "Crops", domain_desc: "Total", - geographicAreas: ["State", "County"], - years: { - "County": allYears, - "State": allYears - } + geographicAreas: ["State", "County"] }; export const queryData: Array = [ @@ -133,7 +129,7 @@ export const queryData: Array = [ geographicAreas: ["State"], years: { "County": [], - "State": allYears.filter(y => Number(y) >= 1998) + "State": allYears.filter(y => Number(y) >= 1987) } }, { @@ -213,6 +209,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["CORN, GRAIN - ACRES HARVESTED"], ["Yield"]: ["CORN, GRAIN - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -227,6 +227,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["COTTON - ACRES HARVESTED"], ["Yield"]: ["COTTON - YIELD, MEASURED IN LB / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -241,6 +245,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["GRAPES - ACRES BEARING"], ["Yield"]: ["GRAPES - YIELD, MEASURED IN TONS / ACRE"] }, + years: { + "County": allYears.filter(y => y === "2002" || Number(y) >= 2007), + "State": allYears.filter(y => y === "2002" || Number(y) >= 2007) + }, ...sharedCropHeaders }, { @@ -255,6 +263,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["OATS - ACRES HARVESTED"], ["Yield"]: ["OATS - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -269,6 +281,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["SOYBEANS - ACRES HARVESTED"], ["Yield"]: ["SOYBEANS - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -283,6 +299,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["WHEAT - ACRES HARVESTED"], ["Yield"]: ["WHEAT - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders } ]; diff --git a/src/scripts/api.ts b/src/scripts/api.ts index bba429e..97dd8fb 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -298,70 +298,72 @@ const processAttributeData = async (props: IProcessAttributeData) => { const isMultiStateRegion = queryParams?.geographicAreas[0] === "REGION : MULTI-STATE"; const data = await prepareQueryAndFetchData({isMultiStateRegion, years, geographicLevel, queryParams, attribute, stateArray, cropUnit, selectedOptions, setReqCount}); - items.forEach((item: any) => { - // find all the data items that match this item's location and year - const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel}); - - if (isMultiStateRegion) { - if (item.State !== "Alaska") { - const { Value } = matchingData.length > 0 && matchingData[0]; - const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; - item[codapColumnName] = Value; - } - } else if (attribute === "Acres Operated") { - // special case for handling acres operated, where we have to sum up the values of several data items - const acreTotals: IAcreTotals = { - [strings.oneTo9Acres]: { - [strings.oneTo9Acres]: 0 - }, - [strings.tenTo49Acres]: { - [strings.tenTo49Acres]: 0 - }, - [strings.fiftyTo100Acres]: { - [strings.fiftyTo69Acres]: 0, - [strings.seventyTo99Acres]: 0 - }, - [strings.oneHundredTo500Acres]: { - [strings.oneHundredTo139Acres]: 0, - [strings.oneHundredFortyTo179Acres]: 0, - [strings.oneHundredEightyTo219Acres]: 0, - [strings.twoHundredTwentyTo259Acres]: 0, - [strings.twoHundredSixtyTo499Acres]: 0 - }, - [strings.fiveHundredTo999Acres]: { - [strings.fiveHundredTo999Acres]: 0 - }, - [strings.oneThousandTo5000Acres]: { - [strings.oneThousandTo1999Acres]: 0, - [strings.twoThousandTo4999Acres]: 0 - }, - [strings.fiveThousandOrMoreAcres]: { - [strings.fiveThousandOrMoreAcres]: 0 + // there might be no data returned for the year/attribute/geoLevel combination, in which case we return items unchanged + if (data) { + items.forEach((item: any) => { + // find all the data items that match this item's location and year + const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel}); + if (isMultiStateRegion) { + if (item.State !== "Alaska") { + const { Value } = matchingData.length > 0 && matchingData[0]; + const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; + item[codapColumnName] = Value; + } + } else if (attribute === "Acres Operated") { + // special case for handling acres operated, where we have to sum up the values of several data items + const acreTotals: IAcreTotals = { + [strings.oneTo9Acres]: { + [strings.oneTo9Acres]: 0 + }, + [strings.tenTo49Acres]: { + [strings.tenTo49Acres]: 0 + }, + [strings.fiftyTo100Acres]: { + [strings.fiftyTo69Acres]: 0, + [strings.seventyTo99Acres]: 0 + }, + [strings.oneHundredTo500Acres]: { + [strings.oneHundredTo139Acres]: 0, + [strings.oneHundredFortyTo179Acres]: 0, + [strings.oneHundredEightyTo219Acres]: 0, + [strings.twoHundredTwentyTo259Acres]: 0, + [strings.twoHundredSixtyTo499Acres]: 0 + }, + [strings.fiveHundredTo999Acres]: { + [strings.fiveHundredTo999Acres]: 0 + }, + [strings.oneThousandTo5000Acres]: { + [strings.oneThousandTo1999Acres]: 0, + [strings.twoThousandTo4999Acres]: 0 + }, + [strings.fiveThousandOrMoreAcres]: { + [strings.fiveThousandOrMoreAcres]: 0 + } + }; + const totalKeys = Object.keys(acreTotals); + for (const total of totalKeys) { + const subTotalKeys = Object.keys(acreTotals[total]); + const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); + subTotalData.forEach((dataObj: any) => { + acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + }); + const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; + // sum up all the values of acreTotals[total] + const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); + const sum = onlyNumbers.reduce((acc, cur) => acc + cur); + item[codapColumnName] = sum; } - }; - const totalKeys = Object.keys(acreTotals); - for (const total of totalKeys) { - const subTotalKeys = Object.keys(acreTotals[total]); - const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); - subTotalData.forEach((dataObj: any) => { - acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + } else { + matchingData.forEach((dataItem: any) => { + const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; + const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; + if (codapColumnName) { + item[codapColumnName] = dataItem.Value; + } }); - const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; - // sum up all the values of acreTotals[total] - const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); - const sum = onlyNumbers.reduce((acc, cur) => acc + cur); - item[codapColumnName] = sum; } - } else { - matchingData.forEach((dataItem: any) => { - const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; - const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; - if (codapColumnName) { - item[codapColumnName] = dataItem.Value; - } - }); - } - }); + }); + } return items; }; @@ -413,7 +415,6 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp reqParams.cropUnits = cropUnits; } const req = createRequest(reqParams); - if (attribute === "Total Farmers" && (years.length > 1 && years.includes("2017"))) { // we need to make two requests -- one for the total farmers in 2017, and one for the total farmers in all other years const req2017 = createRequest({...reqParams, years: ["2017"]}); @@ -423,7 +424,9 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp if (res2017 && resOtherYears) { return [...res2017.data, ...resOtherYears.data]; } else { - console.log(`Error: did not receive resposnse for this request:`, req); + // eslint-disable-next-line no-console + console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`); + return undefined; } } else { const res = await fetchDataWithRetry(req, setReqCount); @@ -431,7 +434,8 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp return res.data; } else { // eslint-disable-next-line no-console - console.log(`Error: did not receive response for this request:`, req); + console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`); + return undefined; } } }; @@ -453,5 +457,6 @@ export const fetchDataWithRetry = async (req: string, setReqCount: ISetReqCount, retries++; } } - throw new Error(`Request failed after ${maxRetries} attempts`); + // throw new Error(`Request failed after ${maxRetries} attempts`); + return undefined; }; From 6d83bba0011e6b23400bd584458d5e22111a18c9 Mon Sep 17 00:00:00 2001 From: lublagg Date: Mon, 12 Feb 2024 18:31:19 -0500 Subject: [PATCH 3/4] Check if matchingData has length. --- src/scripts/api.ts | 129 +++++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 63 deletions(-) diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 97dd8fb..470ca93 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -303,65 +303,66 @@ const processAttributeData = async (props: IProcessAttributeData) => { items.forEach((item: any) => { // find all the data items that match this item's location and year const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel}); - if (isMultiStateRegion) { - if (item.State !== "Alaska") { - const { Value } = matchingData.length > 0 && matchingData[0]; - const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; - item[codapColumnName] = Value; - } - } else if (attribute === "Acres Operated") { - // special case for handling acres operated, where we have to sum up the values of several data items - const acreTotals: IAcreTotals = { - [strings.oneTo9Acres]: { - [strings.oneTo9Acres]: 0 - }, - [strings.tenTo49Acres]: { - [strings.tenTo49Acres]: 0 - }, - [strings.fiftyTo100Acres]: { - [strings.fiftyTo69Acres]: 0, - [strings.seventyTo99Acres]: 0 - }, - [strings.oneHundredTo500Acres]: { - [strings.oneHundredTo139Acres]: 0, - [strings.oneHundredFortyTo179Acres]: 0, - [strings.oneHundredEightyTo219Acres]: 0, - [strings.twoHundredTwentyTo259Acres]: 0, - [strings.twoHundredSixtyTo499Acres]: 0 - }, - [strings.fiveHundredTo999Acres]: { - [strings.fiveHundredTo999Acres]: 0 - }, - [strings.oneThousandTo5000Acres]: { - [strings.oneThousandTo1999Acres]: 0, - [strings.twoThousandTo4999Acres]: 0 - }, - [strings.fiveThousandOrMoreAcres]: { - [strings.fiveThousandOrMoreAcres]: 0 + if (matchingData.length) { + if (isMultiStateRegion) { + const { Value } = matchingData[0]; + const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; + item[codapColumnName] = Value; + } else if (attribute === "Acres Operated") { + // special case for handling acres operated, where we have to sum up the values of several data items + const acreTotals: IAcreTotals = { + [strings.oneTo9Acres]: { + [strings.oneTo9Acres]: 0 + }, + [strings.tenTo49Acres]: { + [strings.tenTo49Acres]: 0 + }, + [strings.fiftyTo100Acres]: { + [strings.fiftyTo69Acres]: 0, + [strings.seventyTo99Acres]: 0 + }, + [strings.oneHundredTo500Acres]: { + [strings.oneHundredTo139Acres]: 0, + [strings.oneHundredFortyTo179Acres]: 0, + [strings.oneHundredEightyTo219Acres]: 0, + [strings.twoHundredTwentyTo259Acres]: 0, + [strings.twoHundredSixtyTo499Acres]: 0 + }, + [strings.fiveHundredTo999Acres]: { + [strings.fiveHundredTo999Acres]: 0 + }, + [strings.oneThousandTo5000Acres]: { + [strings.oneThousandTo1999Acres]: 0, + [strings.twoThousandTo4999Acres]: 0 + }, + [strings.fiveThousandOrMoreAcres]: { + [strings.fiveThousandOrMoreAcres]: 0 + } + }; + const totalKeys = Object.keys(acreTotals); + for (const total of totalKeys) { + const subTotalKeys = Object.keys(acreTotals[total]); + const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); + subTotalData.forEach((dataObj: any) => { + acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + }); + const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; + // sum up all the values of acreTotals[total] + const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); + const sum = onlyNumbers.reduce((acc, cur) => acc + cur); + item[codapColumnName] = sum; } - }; - const totalKeys = Object.keys(acreTotals); - for (const total of totalKeys) { - const subTotalKeys = Object.keys(acreTotals[total]); - const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); - subTotalData.forEach((dataObj: any) => { - acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + } else { + matchingData.forEach((dataItem: any) => { + const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; + const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; + if (codapColumnName) { + item[codapColumnName] = dataItem.Value; + } }); - const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; - // sum up all the values of acreTotals[total] - const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); - const sum = onlyNumbers.reduce((acc, cur) => acc + cur); - item[codapColumnName] = sum; } - } else { - matchingData.forEach((dataItem: any) => { - const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; - const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; - if (codapColumnName) { - item[codapColumnName] = dataItem.Value; - } - }); } + }); } return items; @@ -415,6 +416,7 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp reqParams.cropUnits = cropUnits; } const req = createRequest(reqParams); + console.log("req", req); if (attribute === "Total Farmers" && (years.length > 1 && years.includes("2017"))) { // we need to make two requests -- one for the total farmers in 2017, and one for the total farmers in all other years const req2017 = createRequest({...reqParams, years: ["2017"]}); @@ -430,6 +432,7 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp } } else { const res = await fetchDataWithRetry(req, setReqCount); + console.log("res", res); if (res) { return res.data; } else { @@ -443,20 +446,20 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp export const fetchDataWithRetry = async (req: string, setReqCount: ISetReqCount, maxRetries = 3,) => { let retries = 0; while (retries < maxRetries) { - try { - const response = await fetchJsonp(req, { timeout: 30000 }); // Increase the timeout + const response = await fetchJsonp(req, { timeout: 10000 }); // Increase the timeout + if (response?.ok) { const json = await response.json(); + console.log("json", json); setReqCount((prevState) => { const completed = prevState.completed + 1 > prevState.total ? prevState.total : prevState.completed + 1; return {...prevState, completed}; - }); + }); return json; - } catch (error) { - // eslint-disable-next-line no-console - console.log(`Request attempt ${retries + 1} failed:`, error); - retries++; + } else { + // eslint-disable-next-line no-console + console.log(`Request attempt ${retries + 1} failed:`, req); + retries++; } } - // throw new Error(`Request failed after ${maxRetries} attempts`); return undefined; }; From b4b81c82f202c0898917a7867578144811748c0b Mon Sep 17 00:00:00 2001 From: lublagg Date: Mon, 12 Feb 2024 18:31:26 -0500 Subject: [PATCH 4/4] Remove console.logs --- src/scripts/api.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 470ca93..d8a3f98 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -416,7 +416,6 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp reqParams.cropUnits = cropUnits; } const req = createRequest(reqParams); - console.log("req", req); if (attribute === "Total Farmers" && (years.length > 1 && years.includes("2017"))) { // we need to make two requests -- one for the total farmers in 2017, and one for the total farmers in all other years const req2017 = createRequest({...reqParams, years: ["2017"]}); @@ -432,7 +431,6 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp } } else { const res = await fetchDataWithRetry(req, setReqCount); - console.log("res", res); if (res) { return res.data; } else { @@ -449,7 +447,6 @@ export const fetchDataWithRetry = async (req: string, setReqCount: ISetReqCount, const response = await fetchJsonp(req, { timeout: 10000 }); // Increase the timeout if (response?.ok) { const json = await response.json(); - console.log("json", json); setReqCount((prevState) => { const completed = prevState.completed + 1 > prevState.total ? prevState.total : prevState.completed + 1; return {...prevState, completed};