diff --git a/src/constants/queryHeaders.ts b/src/constants/queryHeaders.ts index a0f1b2b..b92bba9 100644 --- a/src/constants/queryHeaders.ts +++ b/src/constants/queryHeaders.ts @@ -30,11 +30,7 @@ const sharedLaborHeaders = { const sharedCropHeaders = { sect_desc: "Crops", domain_desc: "Total", - geographicAreas: ["State", "County"], - years: { - "County": allYears, - "State": allYears - } + geographicAreas: ["State", "County"] }; export const queryData: Array = [ @@ -133,7 +129,7 @@ export const queryData: Array = [ geographicAreas: ["State"], years: { "County": [], - "State": allYears.filter(y => Number(y) >= 1998) + "State": allYears.filter(y => Number(y) >= 1987) } }, { @@ -213,6 +209,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["CORN, GRAIN - ACRES HARVESTED"], ["Yield"]: ["CORN, GRAIN - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -227,6 +227,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["COTTON - ACRES HARVESTED"], ["Yield"]: ["COTTON - YIELD, MEASURED IN LB / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -241,6 +245,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["GRAPES - ACRES BEARING"], ["Yield"]: ["GRAPES - YIELD, MEASURED IN TONS / ACRE"] }, + years: { + "County": allYears.filter(y => y === "2002" || Number(y) >= 2007), + "State": allYears.filter(y => y === "2002" || Number(y) >= 2007) + }, ...sharedCropHeaders }, { @@ -255,6 +263,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["OATS - ACRES HARVESTED"], ["Yield"]: ["OATS - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -269,6 +281,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["SOYBEANS - ACRES HARVESTED"], ["Yield"]: ["SOYBEANS - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders }, { @@ -283,6 +299,10 @@ export const queryData: Array = [ ["Area Harvested"]: ["WHEAT - ACRES HARVESTED"], ["Yield"]: ["WHEAT - YIELD, MEASURED IN BU / ACRE"] }, + years: { + "County": allYears, + "State": allYears + }, ...sharedCropHeaders } ]; diff --git a/src/scripts/api.ts b/src/scripts/api.ts index bba429e..d8a3f98 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -298,70 +298,73 @@ const processAttributeData = async (props: IProcessAttributeData) => { const isMultiStateRegion = queryParams?.geographicAreas[0] === "REGION : MULTI-STATE"; const data = await prepareQueryAndFetchData({isMultiStateRegion, years, geographicLevel, queryParams, attribute, stateArray, cropUnit, selectedOptions, setReqCount}); - items.forEach((item: any) => { - // find all the data items that match this item's location and year - const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel}); - - if (isMultiStateRegion) { - if (item.State !== "Alaska") { - const { Value } = matchingData.length > 0 && matchingData[0]; - const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; - item[codapColumnName] = Value; - } - } else if (attribute === "Acres Operated") { - // special case for handling acres operated, where we have to sum up the values of several data items - const acreTotals: IAcreTotals = { - [strings.oneTo9Acres]: { - [strings.oneTo9Acres]: 0 - }, - [strings.tenTo49Acres]: { - [strings.tenTo49Acres]: 0 - }, - [strings.fiftyTo100Acres]: { - [strings.fiftyTo69Acres]: 0, - [strings.seventyTo99Acres]: 0 - }, - [strings.oneHundredTo500Acres]: { - [strings.oneHundredTo139Acres]: 0, - [strings.oneHundredFortyTo179Acres]: 0, - [strings.oneHundredEightyTo219Acres]: 0, - [strings.twoHundredTwentyTo259Acres]: 0, - [strings.twoHundredSixtyTo499Acres]: 0 - }, - [strings.fiveHundredTo999Acres]: { - [strings.fiveHundredTo999Acres]: 0 - }, - [strings.oneThousandTo5000Acres]: { - [strings.oneThousandTo1999Acres]: 0, - [strings.twoThousandTo4999Acres]: 0 - }, - [strings.fiveThousandOrMoreAcres]: { - [strings.fiveThousandOrMoreAcres]: 0 + // there might be no data returned for the year/attribute/geoLevel combination, in which case we return items unchanged + if (data) { + items.forEach((item: any) => { + // find all the data items that match this item's location and year + const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel}); + if (matchingData.length) { + if (isMultiStateRegion) { + const { Value } = matchingData[0]; + const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable; + item[codapColumnName] = Value; + } else if (attribute === "Acres Operated") { + // special case for handling acres operated, where we have to sum up the values of several data items + const acreTotals: IAcreTotals = { + [strings.oneTo9Acres]: { + [strings.oneTo9Acres]: 0 + }, + [strings.tenTo49Acres]: { + [strings.tenTo49Acres]: 0 + }, + [strings.fiftyTo100Acres]: { + [strings.fiftyTo69Acres]: 0, + [strings.seventyTo99Acres]: 0 + }, + [strings.oneHundredTo500Acres]: { + [strings.oneHundredTo139Acres]: 0, + [strings.oneHundredFortyTo179Acres]: 0, + [strings.oneHundredEightyTo219Acres]: 0, + [strings.twoHundredTwentyTo259Acres]: 0, + [strings.twoHundredSixtyTo499Acres]: 0 + }, + [strings.fiveHundredTo999Acres]: { + [strings.fiveHundredTo999Acres]: 0 + }, + [strings.oneThousandTo5000Acres]: { + [strings.oneThousandTo1999Acres]: 0, + [strings.twoThousandTo4999Acres]: 0 + }, + [strings.fiveThousandOrMoreAcres]: { + [strings.fiveThousandOrMoreAcres]: 0 + } + }; + const totalKeys = Object.keys(acreTotals); + for (const total of totalKeys) { + const subTotalKeys = Object.keys(acreTotals[total]); + const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); + subTotalData.forEach((dataObj: any) => { + acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); + }); + const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; + // sum up all the values of acreTotals[total] + const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); + const sum = onlyNumbers.reduce((acc, cur) => acc + cur); + item[codapColumnName] = sum; + } + } else { + matchingData.forEach((dataItem: any) => { + const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; + const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; + if (codapColumnName) { + item[codapColumnName] = dataItem.Value; + } + }); } - }; - const totalKeys = Object.keys(acreTotals); - for (const total of totalKeys) { - const subTotalKeys = Object.keys(acreTotals[total]); - const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc)); - subTotalData.forEach((dataObj: any) => { - acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, ""); - }); - const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable; - // sum up all the values of acreTotals[total] - const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k])); - const sum = onlyNumbers.reduce((acc, cur) => acc + cur); - item[codapColumnName] = sum; } - } else { - matchingData.forEach((dataItem: any) => { - const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc; - const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable; - if (codapColumnName) { - item[codapColumnName] = dataItem.Value; - } - }); - } - }); + + }); + } return items; }; @@ -413,7 +416,6 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp reqParams.cropUnits = cropUnits; } const req = createRequest(reqParams); - if (attribute === "Total Farmers" && (years.length > 1 && years.includes("2017"))) { // we need to make two requests -- one for the total farmers in 2017, and one for the total farmers in all other years const req2017 = createRequest({...reqParams, years: ["2017"]}); @@ -423,7 +425,9 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp if (res2017 && resOtherYears) { return [...res2017.data, ...resOtherYears.data]; } else { - console.log(`Error: did not receive resposnse for this request:`, req); + // eslint-disable-next-line no-console + console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`); + return undefined; } } else { const res = await fetchDataWithRetry(req, setReqCount); @@ -431,7 +435,8 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp return res.data; } else { // eslint-disable-next-line no-console - console.log(`Error: did not receive response for this request:`, req); + console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`); + return undefined; } } }; @@ -439,19 +444,19 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp export const fetchDataWithRetry = async (req: string, setReqCount: ISetReqCount, maxRetries = 3,) => { let retries = 0; while (retries < maxRetries) { - try { - const response = await fetchJsonp(req, { timeout: 30000 }); // Increase the timeout + const response = await fetchJsonp(req, { timeout: 10000 }); // Increase the timeout + if (response?.ok) { const json = await response.json(); setReqCount((prevState) => { const completed = prevState.completed + 1 > prevState.total ? prevState.total : prevState.completed + 1; return {...prevState, completed}; - }); + }); return json; - } catch (error) { - // eslint-disable-next-line no-console - console.log(`Request attempt ${retries + 1} failed:`, error); - retries++; + } else { + // eslint-disable-next-line no-console + console.log(`Request attempt ${retries + 1} failed:`, req); + retries++; } } - throw new Error(`Request failed after ${maxRetries} attempts`); + return undefined; };