Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix fetch errors if there is no data for selected combinations of options (PT-186340805) #29

Merged
merged 3 commits into from
Feb 15, 2024
Merged

Fix fetch errors if there is no data for selected combinations of options (PT-186340805) #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
de65ec8
Return undefined for data if bad request.
lublagg Feb 12, 2024
6d83bba
Check if matchingData has length.
lublagg Feb 12, 2024
b4b81c8
Remove console.logs
lublagg Feb 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 26 additions & 6 deletions src/constants/queryHeaders.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,7 @@ const sharedLaborHeaders = {
const sharedCropHeaders = {
sect_desc: "Crops",
domain_desc: "Total",
geographicAreas: ["State", "County"],
years: {
"County": allYears,
"State": allYears
}
geographicAreas: ["State", "County"]
};

export const queryData: Array<IQueryHeaders> = [
Expand Down Expand Up @@ -133,7 +129,7 @@ export const queryData: Array<IQueryHeaders> = [
geographicAreas: ["State"],
years: {
"County": [],
"State": allYears.filter(y => Number(y) >= 1998)
"State": allYears.filter(y => Number(y) >= 1987)
}
},
{
Expand Down Expand Up @@ -213,6 +209,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["CORN, GRAIN - ACRES HARVESTED"],
["Yield"]: ["CORN, GRAIN - YIELD, MEASURED IN BU / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
},
{
Expand All @@ -227,6 +227,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["COTTON - ACRES HARVESTED"],
["Yield"]: ["COTTON - YIELD, MEASURED IN LB / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
},
{
Expand All @@ -241,6 +245,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["GRAPES - ACRES BEARING"],
["Yield"]: ["GRAPES - YIELD, MEASURED IN TONS / ACRE"]
},
years: {
"County": allYears.filter(y => y === "2002" || Number(y) >= 2007),
"State": allYears.filter(y => y === "2002" || Number(y) >= 2007)
},
...sharedCropHeaders
},
{
Expand All @@ -255,6 +263,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["OATS - ACRES HARVESTED"],
["Yield"]: ["OATS - YIELD, MEASURED IN BU / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
},
{
Expand All @@ -269,6 +281,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["SOYBEANS - ACRES HARVESTED"],
["Yield"]: ["SOYBEANS - YIELD, MEASURED IN BU / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
},
{
Expand All @@ -283,6 +299,10 @@ export const queryData: Array<IQueryHeaders> = [
["Area Harvested"]: ["WHEAT - ACRES HARVESTED"],
["Yield"]: ["WHEAT - YIELD, MEASURED IN BU / ACRE"]
},
years: {
"County": allYears,
"State": allYears
},
...sharedCropHeaders
}
];
151 changes: 78 additions & 73 deletions src/scripts/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,70 +298,73 @@ const processAttributeData = async (props: IProcessAttributeData) => {
const isMultiStateRegion = queryParams?.geographicAreas[0] === "REGION : MULTI-STATE";
const data = await prepareQueryAndFetchData({isMultiStateRegion, years, geographicLevel, queryParams, attribute, stateArray, cropUnit, selectedOptions, setReqCount});

items.forEach((item: any) => {
// find all the data items that match this item's location and year
const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel});

if (isMultiStateRegion) {
if (item.State !== "Alaska") {
const { Value } = matchingData.length > 0 && matchingData[0];
const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable;
item[codapColumnName] = Value;
}
} else if (attribute === "Acres Operated") {
// special case for handling acres operated, where we have to sum up the values of several data items
const acreTotals: IAcreTotals = {
[strings.oneTo9Acres]: {
[strings.oneTo9Acres]: 0
},
[strings.tenTo49Acres]: {
[strings.tenTo49Acres]: 0
},
[strings.fiftyTo100Acres]: {
[strings.fiftyTo69Acres]: 0,
[strings.seventyTo99Acres]: 0
},
[strings.oneHundredTo500Acres]: {
[strings.oneHundredTo139Acres]: 0,
[strings.oneHundredFortyTo179Acres]: 0,
[strings.oneHundredEightyTo219Acres]: 0,
[strings.twoHundredTwentyTo259Acres]: 0,
[strings.twoHundredSixtyTo499Acres]: 0
},
[strings.fiveHundredTo999Acres]: {
[strings.fiveHundredTo999Acres]: 0
},
[strings.oneThousandTo5000Acres]: {
[strings.oneThousandTo1999Acres]: 0,
[strings.twoThousandTo4999Acres]: 0
},
[strings.fiveThousandOrMoreAcres]: {
[strings.fiveThousandOrMoreAcres]: 0
// there might be no data returned for the year/attribute/geoLevel combination, in which case we return items unchanged
if (data) {
items.forEach((item: any) => {
// find all the data items that match this item's location and year
const matchingData = findMatchingData({isMultiStateRegion, data, item, geoLevel: geographicLevel});
if (matchingData.length) {
if (isMultiStateRegion) {
const { Value } = matchingData[0];
const codapColumnName = attrToCODAPColumnName[matchingData[0].short_desc].attributeNameInCodapTable;
item[codapColumnName] = Value;
} else if (attribute === "Acres Operated") {
// special case for handling acres operated, where we have to sum up the values of several data items
const acreTotals: IAcreTotals = {
[strings.oneTo9Acres]: {
[strings.oneTo9Acres]: 0
},
[strings.tenTo49Acres]: {
[strings.tenTo49Acres]: 0
},
[strings.fiftyTo100Acres]: {
[strings.fiftyTo69Acres]: 0,
[strings.seventyTo99Acres]: 0
},
[strings.oneHundredTo500Acres]: {
[strings.oneHundredTo139Acres]: 0,
[strings.oneHundredFortyTo179Acres]: 0,
[strings.oneHundredEightyTo219Acres]: 0,
[strings.twoHundredTwentyTo259Acres]: 0,
[strings.twoHundredSixtyTo499Acres]: 0
},
[strings.fiveHundredTo999Acres]: {
[strings.fiveHundredTo999Acres]: 0
},
[strings.oneThousandTo5000Acres]: {
[strings.oneThousandTo1999Acres]: 0,
[strings.twoThousandTo4999Acres]: 0
},
[strings.fiveThousandOrMoreAcres]: {
[strings.fiveThousandOrMoreAcres]: 0
}
};
const totalKeys = Object.keys(acreTotals);
for (const total of totalKeys) {
const subTotalKeys = Object.keys(acreTotals[total]);
const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc));
subTotalData.forEach((dataObj: any) => {
acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, "");
});
const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable;
// sum up all the values of acreTotals[total]
const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k]));
const sum = onlyNumbers.reduce((acc, cur) => acc + cur);
item[codapColumnName] = sum;
}
} else {
matchingData.forEach((dataItem: any) => {
const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc;
const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable;
if (codapColumnName) {
item[codapColumnName] = dataItem.Value;
}
});
}
};
const totalKeys = Object.keys(acreTotals);
for (const total of totalKeys) {
const subTotalKeys = Object.keys(acreTotals[total]);
const subTotalData = matchingData.filter((dataItem: any) => subTotalKeys.includes(dataItem.domaincat_desc));
subTotalData.forEach((dataObj: any) => {
acreTotals[total][dataObj.domaincat_desc] = dataObj.Value.replace(/,/g, "");
});
const codapColumnName = attrToCODAPColumnName[total].attributeNameInCodapTable;
// sum up all the values of acreTotals[total]
const onlyNumbers = subTotalKeys.map((k) => Number(acreTotals[total][k]));
const sum = onlyNumbers.reduce((acc, cur) => acc + cur);
item[codapColumnName] = sum;
}
} else {
matchingData.forEach((dataItem: any) => {
const dataItemDesc = attribute === "Economic Class" ? dataItem.domaincat_desc : dataItem.short_desc;
const codapColumnName = attrToCODAPColumnName[dataItemDesc]?.attributeNameInCodapTable;
if (codapColumnName) {
item[codapColumnName] = dataItem.Value;
}
});
}
});

});
}
return items;
};

Expand Down Expand Up @@ -413,7 +416,6 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp
reqParams.cropUnits = cropUnits;
}
const req = createRequest(reqParams);

if (attribute === "Total Farmers" && (years.length > 1 && years.includes("2017"))) {
// we need to make two requests -- one for the total farmers in 2017, and one for the total farmers in all other years
const req2017 = createRequest({...reqParams, years: ["2017"]});
Expand All @@ -423,35 +425,38 @@ const getAttrData = async (params: IGetAttrDataParams, selectedOptions: IStateOp
if (res2017 && resOtherYears) {
return [...res2017.data, ...resOtherYears.data];
} else {
console.log(`Error: did not receive resposnse for this request:`, req);
// eslint-disable-next-line no-console
console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`);
return undefined;
}
} else {
const res = await fetchDataWithRetry(req, setReqCount);
if (res) {
return res.data;
} else {
// eslint-disable-next-line no-console
console.log(`Error: did not receive response for this request:`, req);
console.log(`No data returned for ${attribute} at ${geographicLevel} level in ${years} for ${states}`);
return undefined;
}
}
};

export const fetchDataWithRetry = async (req: string, setReqCount: ISetReqCount, maxRetries = 3,) => {
let retries = 0;
while (retries < maxRetries) {
try {
const response = await fetchJsonp(req, { timeout: 30000 }); // Increase the timeout
const response = await fetchJsonp(req, { timeout: 10000 }); // Increase the timeout
if (response?.ok) {
const json = await response.json();
setReqCount((prevState) => {
const completed = prevState.completed + 1 > prevState.total ? prevState.total : prevState.completed + 1;
return {...prevState, completed};
});
});
return json;
} catch (error) {
// eslint-disable-next-line no-console
console.log(`Request attempt ${retries + 1} failed:`, error);
retries++;
} else {
// eslint-disable-next-line no-console
console.log(`Request attempt ${retries + 1} failed:`, req);
retries++;
}
}
throw new Error(`Request failed after ${maxRetries} attempts`);
return undefined;
};
Loading