From de65ec862f2bd190169eca6e2bf6166ca9e7c746 Mon Sep 17 00:00:00 2001 From: lublagg Date: Mon, 12 Feb 2024 18:14:03 -0500 Subject: [PATCH 1/3] Return undefined for data if bad request. --- src/constants/queryHeaders.ts | 32 ++++++-- src/scripts/api.ts | 135 ++++++++++++++++++---------------- 2 files changed, 96 insertions(+), 71 deletions(-) diff --git a/src/constants/queryHeaders.ts b/src/constants/queryHeaders.ts index a0f1b2b..b92bba9 100644 --- a/src/constants/queryHeaders.ts +++ b/src/constants/queryHeaders.ts @@ -30,11 +30,7 @@ const sharedLaborHeaders = { const sharedCropHeaders = { sect_desc: "Crops", domain_desc: "Total", - geographicAreas: ["State", "County"], - years: { - "County": allYears, - "State": allYears - } + geographicAreas: ["State", "County"] }; export const queryData: Array = [ @@ -133,7 +129,7 @@ export const queryData: Array = [ geographicAreas: ["State"], years: { "County": [], - "State": allYears.filter(y => Number(y) >= 1998) + "State": allYears.filter(y => Number(y) >= 1987) } }, { @@ -213,6 +209,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["CORN, GRAIN - ACRES HARVESTED"], ["Yield"]: ["CORN, GRAIN - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -227,6 +227,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["COTTON - ACRES HARVESTED"], ["Yield"]: ["COTTON - YIELD, MEASURED IN LB / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -241,6 +245,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["GRAPES - ACRES BEARING"], ["Yield"]: ["GRAPES - YIELD, MEASURED IN TONS / ACRE"] }, + years: { + "County": allYears.filter(y => y === "2002" || Number(y) >= 2007), + "State": allYears.filter(y => y === "2002" || Number(y) >= 2007) + }, ...sharedCropHeaders }, { @@ -255,6 +263,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["OATS - ACRES HARVESTED"], ["Yield"]: ["OATS - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -269,6 +281,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["SOYBEANS - ACRES HARVESTED"], ["Yield"]: ["SOYBEANS - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -283,6 +299,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["WHEAT - ACRES HARVESTED"], ["Yield"]: ["WHEAT - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders } ]; diff --git a/src/scripts/api.ts b/src/scripts/api.ts index bba429e..97dd8fb 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -298,70 +298,72 @@ const processAttributeData = async (props: IProcessAttributeData) => { const isMultiStateRegion = queryParams?.geographicAreas[0] === "REGION : MULTI-STATE"; const data = await prepareQueryAndFetchData({isMultiStateRegion, years, geographicLevel, queryParams, attribute, stateArray, cropUnit, selectedOptions, setReqCount}); - items.forEach((item: any) => { - // find all the data items that match this item's location and year - const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel}); - - if (isMultiStateRegion) { - if (item.State !== "Alaska") { - const { Value } = matchingData.length > 0 && matchingData[0]; - const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; - item[codapColumnName] = Value; - } - } else if (attribute === "Acres Operated") { - // special case for handling acres operated, where we have to sum up the values of several data items - const acreTotals: IAcreTotals = { - [strings.oneTo9Acres]: { - [strings.oneTo9Acres]: 0 - }, - [strings.tenTo49Acres]: { - [strings.tenTo49Acres]: 0 - }, - [strings.fiftyTo100Acres]: { - [strings.fiftyTo69Acres]: 0, - [strings.seventyTo99Acres]: 0 - }, - [strings.oneHundredTo500Acres]: { - [strings.oneHundredTo139Acres]: 0, - [strings.oneHundredFortyTo179Acres]: 0, - [strings.oneHundredEightyTo219Acres]: 0, - [strings.twoHundredTwentyTo259Acres]: 0, - [strings.twoHundredSixtyTo499Acres]: 0 - }, - [strings.fiveHundredTo999Acres]: { - [strings.fiveHundredTo999Acres]: 0 - }, - [strings.oneThousandTo5000Acres]: { - [strings.oneThousandTo1999Acres]: 0, - [strings.twoThousandTo4999Acres]: 0 - }, - [strings.fiveThousandOrMoreAcres]: { - [strings.fiveThousandOrMoreAcres]: 0 + // there might be no data returned for the year/attribute/geoLevel combination, in which case we return items unchanged + if (data) { + items.forEach((item: any) => { + // find all the data items that match this item's location and year + const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel}); + if (isMultiStateRegion) { + if (item.State !== "Alaska") { + const { Value } = matchingData.length > 0 && matchingData[0]; + const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; + item[codapColumnName] = Value; + } + } else if (attribute === "Acres Operated") { + // special case for handling acres operated, where we have to sum up the values of several data items + const acreTotals: IAcreTotals = { + [strings.oneTo9Acres]: { + [strings.oneTo9Acres]: 0 + }, + [strings.tenTo49Acres]: { + [strings.tenTo49Acres]: 0 + }, + [strings.fiftyTo100Acres]: { + [strings.fiftyTo69Acres]: 0, + [strings.seventyTo99Acres]: 0 + }, + [strings.oneHundredTo500Acres]: { + [strings.oneHundredTo139Acres]: 0, + [strings.oneHundredFortyTo179Acres]: 0, + [strings.oneHundredEightyTo219Acres]: 0, + [strings.twoHundredTwentyTo259Acres]: 0, + [strings.twoHundredSixtyTo499Acres]: 0 + }, + [strings.fiveHundredTo999Acres]: { + [strings.fiveHundredTo999Acres]: 0 + }, + [strings.oneThousandTo5000Acres]: { + [strings.oneThousandTo1999Acres]: 0, + [strings.twoThousandTo4999Acres]: 0 + }, + [strings.fiveThousandOrMoreAcres]: { + [strings.fiveThousandOrMoreAcres]: 0 + } + }; + const totalKeys = Object.keys(acreTotals); + for (const total of totalKeys) { + const subTotalKeys = Object.keys(acreTotals[total]); + const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); + subTotalData.forEach((dataObj: any) => { + acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + }); + const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; + // sum up all the values of acreTotals[total] + const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); + const sum = onlyNumbers.reduce((acc, cur) => acc + cur); + item[codapColumnName] = sum; } - }; - const totalKeys = Object.keys(acreTotals); - for (const total of totalKeys) { - const subTotalKeys = Object.keys(acreTotals[total]); - const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); - subTotalData.forEach((dataObj: any) => { - acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + } else { + matchingData.forEach((dataItem: any) => { + const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; + const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; + if (codapColumnName) { + item[codapColumnName] = dataItem.Value; + } }); - const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; - // sum up all the values of acreTotals[total] - const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); - const sum = onlyNumbers.reduce((acc, cur) => acc + cur); - item[codapColumnName] = sum; } - } else { - matchingData.forEach((dataItem: any) => { - const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; - const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; - if (codapColumnName) { - item[codapColumnName] = dataItem.Value; - } - }); - } - }); + }); + } return items; }; @@ -413,7 +415,6 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp reqParams.cropUnits = cropUnits; } const req = createRequest(reqParams); - if (attribute === "Total Farmers" && (years.length > 1 && years.includes("2017"))) { // we need to make two requests -- one for the total farmers in 2017, and one for the total farmers in all other years const req2017 = createRequest({...reqParams, years: ["2017"]}); @@ -423,7 +424,9 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp if (res2017 && resOtherYears) { return [...res2017.data, ...resOtherYears.data]; } else { - console.log(`Error: did not receive resposnse for this request:`, req); + // eslint-disable-next-line no-console + console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`); + return undefined; } } else { const res = await fetchDataWithRetry(req, setReqCount); @@ -431,7 +434,8 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp return res.data; } else { // eslint-disable-next-line no-console - console.log(`Error: did not receive response for this request:`, req); + console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`); + return undefined; } } }; @@ -453,5 +457,6 @@ export const fetchDataWithRetry = async (req: string, setReqCount: ISetReqCount, retries++; } } - throw new Error(`Request failed after ${maxRetries} attempts`); + // throw new Error(`Request failed after ${maxRetries} attempts`); + return undefined; }; From 6d83bba0011e6b23400bd584458d5e22111a18c9 Mon Sep 17 00:00:00 2001 From: lublagg Date: Mon, 12 Feb 2024 18:31:19 -0500 Subject: [PATCH 2/3] Check if matchingData has length. --- src/scripts/api.ts | 129 +++++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 63 deletions(-) diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 97dd8fb..470ca93 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -303,65 +303,66 @@ const processAttributeData = async (props: IProcessAttributeData) => { items.forEach((item: any) => { // find all the data items that match this item's location and year const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel}); - if (isMultiStateRegion) { - if (item.State !== "Alaska") { - const { Value } = matchingData.length > 0 && matchingData[0]; - const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; - item[codapColumnName] = Value; - } - } else if (attribute === "Acres Operated") { - // special case for handling acres operated, where we have to sum up the values of several data items - const acreTotals: IAcreTotals = { - [strings.oneTo9Acres]: { - [strings.oneTo9Acres]: 0 - }, - [strings.tenTo49Acres]: { - [strings.tenTo49Acres]: 0 - }, - [strings.fiftyTo100Acres]: { - [strings.fiftyTo69Acres]: 0, - [strings.seventyTo99Acres]: 0 - }, - [strings.oneHundredTo500Acres]: { - [strings.oneHundredTo139Acres]: 0, - [strings.oneHundredFortyTo179Acres]: 0, - [strings.oneHundredEightyTo219Acres]: 0, - [strings.twoHundredTwentyTo259Acres]: 0, - [strings.twoHundredSixtyTo499Acres]: 0 - }, - [strings.fiveHundredTo999Acres]: { - [strings.fiveHundredTo999Acres]: 0 - }, - [strings.oneThousandTo5000Acres]: { - [strings.oneThousandTo1999Acres]: 0, - [strings.twoThousandTo4999Acres]: 0 - }, - [strings.fiveThousandOrMoreAcres]: { - [strings.fiveThousandOrMoreAcres]: 0 + if (matchingData.length) { + if (isMultiStateRegion) { + const { Value } = matchingData[0]; + const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; + item[codapColumnName] = Value; + } else if (attribute === "Acres Operated") { + // special case for handling acres operated, where we have to sum up the values of several data items + const acreTotals: IAcreTotals = { + [strings.oneTo9Acres]: { + [strings.oneTo9Acres]: 0 + }, + [strings.tenTo49Acres]: { + [strings.tenTo49Acres]: 0 + }, + [strings.fiftyTo100Acres]: { + [strings.fiftyTo69Acres]: 0, + [strings.seventyTo99Acres]: 0 + }, + [strings.oneHundredTo500Acres]: { + [strings.oneHundredTo139Acres]: 0, + [strings.oneHundredFortyTo179Acres]: 0, + [strings.oneHundredEightyTo219Acres]: 0, + [strings.twoHundredTwentyTo259Acres]: 0, + [strings.twoHundredSixtyTo499Acres]: 0 + }, + [strings.fiveHundredTo999Acres]: { + [strings.fiveHundredTo999Acres]: 0 + }, + [strings.oneThousandTo5000Acres]: { + [strings.oneThousandTo1999Acres]: 0, + [strings.twoThousandTo4999Acres]: 0 + }, + [strings.fiveThousandOrMoreAcres]: { + [strings.fiveThousandOrMoreAcres]: 0 + } + }; + const totalKeys = Object.keys(acreTotals); + for (const total of totalKeys) { + const subTotalKeys = Object.keys(acreTotals[total]); + const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); + subTotalData.forEach((dataObj: any) => { + acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + }); + const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; + // sum up all the values of acreTotals[total] + const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); + const sum = onlyNumbers.reduce((acc, cur) => acc + cur); + item[codapColumnName] = sum; } - }; - const totalKeys = Object.keys(acreTotals); - for (const total of totalKeys) { - const subTotalKeys = Object.keys(acreTotals[total]); - const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); - subTotalData.forEach((dataObj: any) => { - acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + } else { + matchingData.forEach((dataItem: any) => { + const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; + const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; + if (codapColumnName) { + item[codapColumnName] = dataItem.Value; + } }); - const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; - // sum up all the values of acreTotals[total] - const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); - const sum = onlyNumbers.reduce((acc, cur) => acc + cur); - item[codapColumnName] = sum; } - } else { - matchingData.forEach((dataItem: any) => { - const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; - const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; - if (codapColumnName) { - item[codapColumnName] = dataItem.Value; - } - }); } + }); } return items; @@ -415,6 +416,7 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp reqParams.cropUnits = cropUnits; } const req = createRequest(reqParams); + console.log("req", req); if (attribute === "Total Farmers" && (years.length > 1 && years.includes("2017"))) { // we need to make two requests -- one for the total farmers in 2017, and one for the total farmers in all other years const req2017 = createRequest({...reqParams, years: ["2017"]}); @@ -430,6 +432,7 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp } } else { const res = await fetchDataWithRetry(req, setReqCount); + console.log("res", res); if (res) { return res.data; } else { @@ -443,20 +446,20 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp export const fetchDataWithRetry = async (req: string, setReqCount: ISetReqCount, maxRetries = 3,) => { let retries = 0; while (retries < maxRetries) { - try { - const response = await fetchJsonp(req, { timeout: 30000 }); // Increase the timeout + const response = await fetchJsonp(req, { timeout: 10000 }); // Increase the timeout + if (response?.ok) { const json = await response.json(); + console.log("json", json); setReqCount((prevState) => { const completed = prevState.completed + 1 > prevState.total ? prevState.total : prevState.completed + 1; return {...prevState, completed}; - }); + }); return json; - } catch (error) { - // eslint-disable-next-line no-console - console.log(`Request attempt ${retries + 1} failed:`, error); - retries++; + } else { + // eslint-disable-next-line no-console + console.log(`Request attempt ${retries + 1} failed:`, req); + retries++; } } - // throw new Error(`Request failed after ${maxRetries} attempts`); return undefined; }; From b4b81c82f202c0898917a7867578144811748c0b Mon Sep 17 00:00:00 2001 From: lublagg Date: Mon, 12 Feb 2024 18:31:26 -0500 Subject: [PATCH 3/3] Remove console.logs --- src/scripts/api.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 470ca93..d8a3f98 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -416,7 +416,6 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp reqParams.cropUnits = cropUnits; } const req = createRequest(reqParams); - console.log("req", req); if (attribute === "Total Farmers" && (years.length > 1 && years.includes("2017"))) { // we need to make two requests -- one for the total farmers in 2017, and one for the total farmers in all other years const req2017 = createRequest({...reqParams, years: ["2017"]}); @@ -432,7 +431,6 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp } } else { const res = await fetchDataWithRetry(req, setReqCount); - console.log("res", res); if (res) { return res.data; } else { @@ -449,7 +447,6 @@ export const fetchDataWithRetry = async (req: string, setReqCount: ISetReqCount, const response = await fetchJsonp(req, { timeout: 10000 }); // Increase the timeout if (response?.ok) { const json = await response.json(); - console.log("json", json); setReqCount((prevState) => { const completed = prevState.completed + 1 > prevState.total ? prevState.total : prevState.completed + 1; return {...prevState, completed};