Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 48 additions & 22 deletions utils/hashnode/fetch-from-hashnode.js
Original file line number Diff line number Diff line change
Expand Up @@ -86,28 +86,54 @@ const fetchFromHashnode = async contentType => {
let hasNextPage = true;

while (hasNextPage) {
const res =
eleventyEnv === 'ci' && currentLocale_i18n === 'english'
? require(`../../cypress/fixtures/mock-hashnode-${contentType}.json`)
: await request(process.env.HASHNODE_API_URL, query, {
host: hashnodeHost,
first: 20,
after
});

const resData =
res.publication[fieldName]?.edges.map(({ node }) => node) || [];
const pageInfo = res.publication[fieldName]?.pageInfo;

if (resData.length > 0)
console.log(
`Fetched Hashnode ${contentType} ${pageInfo.endCursor}... and using ${process.memoryUsage.rss() / 1024 / 1024} MB of memory`
);

after = pageInfo.endCursor;
hasNextPage = pageInfo.hasNextPage;

data.push(...resData);
let retries = 3;
let success = false;

while (retries > 0 && !success) {
try {
const res =
eleventyEnv === 'ci' && currentLocale_i18n === 'english'
? require(
`../../cypress/fixtures/mock-hashnode-${contentType}.json`
)
: await request(process.env.HASHNODE_API_URL, query, {
host: hashnodeHost,
first: 20,
after
});

const resData =
res.publication[fieldName]?.edges.map(({ node }) => node) || [];
const pageInfo = res.publication[fieldName]?.pageInfo;

if (resData.length > 0)
console.log(
`Fetched Hashnode ${contentType} ${pageInfo.endCursor}... and using ${process.memoryUsage.rss() / 1024 / 1024} MB of memory`
);

after = pageInfo.endCursor;
hasNextPage = pageInfo.hasNextPage;

data.push(...resData);

success = true;
} catch (error) {
if (error.message.includes('ECONNRESET') && retries > 1) {
console.log(
`Connection reset error. Retrying... (${retries - 1} attempts left)`
);
retries--;
await wait(10000); // Wait for 10 seconds before retrying
} else {
throw error; // If it's not a connection reset or no more retries, rethrow the error
}
}
}

if (!success) {
console.error('Failed to fetch data after multiple retries');
break;
}

await wait(200);
}
Expand Down
Loading