Skip to content

Commit

Permalink
Merge pull request #29 from concord-consortium/186340805-get-data-bugs
Browse files Browse the repository at this point in the history 
Fix fetch errors if there is no data for selected combinations of options (PT-186340805)
  • Loading branch information
@lublagg
lublagg authored Feb 15, 2024
2 parents e8d483e + b4b81c8 commit 40487b1
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 79 deletions.
32 changes: 26 additions & 6 deletions src/constants/queryHeaders.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,7 @@ const sharedLaborHeaders = {
const sharedCropHeaders = {
sect_desc: "Crops",
domain_desc: "Total",
geographicAreas: ["State", "County"],
years: {
"County": allYears,
"State": allYears
}
geographicAreas: ["State", "County"]
};

export const queryData: Array<IQueryHeaders> = [
Expand Down Expand Up @@ -133,7 +129,7 @@ export const queryData: Array<IQueryHeaders> = [
geographicAreas: ["State"],
years: {
"County": [],
"State": allYears.filter(y => Number(y) >= 1998)
"State": allYears.filter(y => Number(y) >= 1987)
}
},
{
Expand Down Expand Up @@ -213,6 +209,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["CORN, GRAIN - ACRES HARVESTED"],
["Yield"]: ["CORN, GRAIN - YIELD, MEASURED IN BU / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
},
{
Expand All @@ -227,6 +227,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["COTTON - ACRES HARVESTED"],
["Yield"]: ["COTTON - YIELD, MEASURED IN LB / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
},
{
Expand All @@ -241,6 +245,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["GRAPES - ACRES BEARING"],
["Yield"]: ["GRAPES - YIELD, MEASURED IN TONS / ACRE"]
},
years: {
"County": allYears.filter(y => y === "2002" || Number(y) >= 2007),
"State": allYears.filter(y => y === "2002" || Number(y) >= 2007)
},
...sharedCropHeaders
},
{
Expand All @@ -255,6 +263,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["OATS - ACRES HARVESTED"],
["Yield"]: ["OATS - YIELD, MEASURED IN BU / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
},
{
Expand All @@ -269,6 +281,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["SOYBEANS - ACRES HARVESTED"],
["Yield"]: ["SOYBEANS - YIELD, MEASURED IN BU / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
},
{
Expand All @@ -283,6 +299,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["WHEAT - ACRES HARVESTED"],
["Yield"]: ["WHEAT - YIELD, MEASURED IN BU / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
}
];
151 changes: 78 additions & 73 deletions src/scripts/api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,70 +298,73 @@ const processAttributeData = async (props: IProcessAttributeData) => {
const isMultiStateRegion = queryParams?.geographicAreas[0] === "REGION : MULTI-STATE";
const data = await prepareQueryAndFetchData({isMultiStateRegion, years, geographicLevel, queryParams, attribute, stateArray, cropUnit, selectedOptions, setReqCount});

items.forEach((item: any) => {
// find all the data items that match this item's location and year
const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel});

if (isMultiStateRegion) {
if (item.State !== "Alaska") {
const { Value } = matchingData.length > 0 && matchingData[0];
const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable;
item[codapColumnName] = Value;
}
} else if (attribute === "Acres Operated") {
// special case for handling acres operated, where we have to sum up the values of several data items
const acreTotals: IAcreTotals = {
[strings.oneTo9Acres]: {
[strings.oneTo9Acres]: 0
},
[strings.tenTo49Acres]: {
[strings.tenTo49Acres]: 0
},
[strings.fiftyTo100Acres]: {
[strings.fiftyTo69Acres]: 0,
[strings.seventyTo99Acres]: 0
},
[strings.oneHundredTo500Acres]: {
[strings.oneHundredTo139Acres]: 0,
[strings.oneHundredFortyTo179Acres]: 0,
[strings.oneHundredEightyTo219Acres]: 0,
[strings.twoHundredTwentyTo259Acres]: 0,
[strings.twoHundredSixtyTo499Acres]: 0
},
[strings.fiveHundredTo999Acres]: {
[strings.fiveHundredTo999Acres]: 0
},
[strings.oneThousandTo5000Acres]: {
[strings.oneThousandTo1999Acres]: 0,
[strings.twoThousandTo4999Acres]: 0
},
[strings.fiveThousandOrMoreAcres]: {
[strings.fiveThousandOrMoreAcres]: 0
// there might be no data returned for the year/attribute/geoLevel combination, in which case we return items unchanged
if (data) {
items.forEach((item: any) => {
// find all the data items that match this item's location and year
const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel});
if (matchingData.length) {
if (isMultiStateRegion) {
const { Value } = matchingData[0];
const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable;
item[codapColumnName] = Value;
} else if (attribute === "Acres Operated") {
// special case for handling acres operated, where we have to sum up the values of several data items
const acreTotals: IAcreTotals = {
[strings.oneTo9Acres]: {
[strings.oneTo9Acres]: 0
},
[strings.tenTo49Acres]: {
[strings.tenTo49Acres]: 0
},
[strings.fiftyTo100Acres]: {
[strings.fiftyTo69Acres]: 0,
[strings.seventyTo99Acres]: 0
},
[strings.oneHundredTo500Acres]: {
[strings.oneHundredTo139Acres]: 0,
[strings.oneHundredFortyTo179Acres]: 0,
[strings.oneHundredEightyTo219Acres]: 0,
[strings.twoHundredTwentyTo259Acres]: 0,
[strings.twoHundredSixtyTo499Acres]: 0
},
[strings.fiveHundredTo999Acres]: {
[strings.fiveHundredTo999Acres]: 0
},
[strings.oneThousandTo5000Acres]: {
[strings.oneThousandTo1999Acres]: 0,
[strings.twoThousandTo4999Acres]: 0
},
[strings.fiveThousandOrMoreAcres]: {
[strings.fiveThousandOrMoreAcres]: 0
}
};
const totalKeys = Object.keys(acreTotals);
for (const total of totalKeys) {
const subTotalKeys = Object.keys(acreTotals[total]);
const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc));
subTotalData.forEach((dataObj: any) => {
acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, "");
});
const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable;
// sum up all the values of acreTotals[total]
const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k]));
const sum = onlyNumbers.reduce((acc, cur) => acc + cur);
item[codapColumnName] = sum;
}
} else {
matchingData.forEach((dataItem: any) => {
const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc;
const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable;
if (codapColumnName) {
item[codapColumnName] = dataItem.Value;
}
});
}
};
const totalKeys = Object.keys(acreTotals);
for (const total of totalKeys) {
const subTotalKeys = Object.keys(acreTotals[total]);
const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc));
subTotalData.forEach((dataObj: any) => {
acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, "");
});
const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable;
// sum up all the values of acreTotals[total]
const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k]));
const sum = onlyNumbers.reduce((acc, cur) => acc + cur);
item[codapColumnName] = sum;
}
} else {
matchingData.forEach((dataItem: any) => {
const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc;
const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable;
if (codapColumnName) {
item[codapColumnName] = dataItem.Value;
}
});
}
});

});
}
return items;
};

Expand Down Expand Up @@ -413,7 +416,6 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp
reqParams.cropUnits = cropUnits;
}
const req = createRequest(reqParams);

if (attribute === "Total Farmers" && (years.length > 1 && years.includes("2017"))) {
// we need to make two requests -- one for the total farmers in 2017, and one for the total farmers in all other years
const req2017 = createRequest({...reqParams, years: ["2017"]});
Expand All @@ -423,35 +425,38 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp
if (res2017 && resOtherYears) {
return [...res2017.data, ...resOtherYears.data];
} else {
console.log(`Error: did not receive resposnse for this request:`, req);
// eslint-disable-next-line no-console
console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`);
return undefined;
}
} else {
const res = await fetchDataWithRetry(req, setReqCount);
if (res) {
return res.data;
} else {
// eslint-disable-next-line no-console
console.log(`Error: did not receive response for this request:`, req);
console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`);
return undefined;
}
}
};

export const fetchDataWithRetry = async (req: string, setReqCount: ISetReqCount, maxRetries = 3,) => {
let retries = 0;
while (retries < maxRetries) {
try {
const response = await fetchJsonp(req, { timeout: 30000 }); // Increase the timeout
const response = await fetchJsonp(req, { timeout: 10000 }); // Increase the timeout
if (response?.ok) {
const json = await response.json();
setReqCount((prevState) => {
const completed = prevState.completed + 1 > prevState.total ? prevState.total : prevState.completed + 1;
return {...prevState, completed};
});
});
return json;
} catch (error) {
// eslint-disable-next-line no-console
console.log(`Request attempt ${retries + 1} failed:`, error);
retries++;
} else {
// eslint-disable-next-line no-console
console.log(`Request attempt ${retries + 1} failed:`, req);
retries++;
}
}
throw new Error(`Request failed after ${maxRetries} attempts`);
return undefined;
};

0 comments on commit 40487b1

Please sign in to comment.