feat: handle non-local files with grace #2135
Comments
SgtPooki
added
exp/intermediate
SgtPooki
moved this from Needs Grooming
to Prioritized / Ready for Dev
in IPFS-GUI (PL EngRes)
To determine if a file is local, I'm using the following commands: ipfs block stat --offline CID # ipfs.block.stat(cid, { offline: true })I’m retrieving all blocks for the current directory and checking if they are local or not by using: ipfs dag get CID # ipfs.dag.get(f.cid)I modified the
/**
* @param {IPFSService} ipfs
* @param {CID} cid
* @param {Object} options
* @param {string} options.path
* @param {boolean} [options.isRoot]
* @param {import('./utils').Sorting} options.sorting
*/
const dirStats = async (ipfs, cid, { path, isRoot, sorting }) => {
const entries = await all(ipfs.ls(cid)) || []
// Workarounds regression in IPFS HTTP Client that causes
// ls on empty dir to return list with that dir only.
// @see https://github.com/ipfs/js-ipfs/issues/3566
const res = (entries.length === 1 && entries[0].cid.toString() === cid.toString())
? []
: entries
const files = []
// precaution: there was a historical performance issue when too many dirs were present
let dirCount = 0
for (const f of res) {
const absPath = join(path, f.name)
let file = null
if (dirCount < 1000 && (f.type === 'directory' || f.type === 'dir')) {
dirCount += 1
file = fileFromStats({ ...await stat(ipfs, f.cid), path: absPath })
} else {
file = fileFromStats({ ...f, path: absPath })
}
// Check if file is local
const refs = await ipfs.dag.get(f.cid)
// Extracting all CIDs
const refsCids = refs.value.Links.map(link => link.Hash['/'])
// console.log(refsCids)
// console.log(`refs=${JSON.stringify(refs, null, 2)}`)
if (dirCount > 300) {
file.isLocal = undefined
} else
if (refsCids.length === 0) {
file.isLocal = true
} else {
file.isLocal = true
// for (const myCid of refsCids) {
const myCid = refsCids[0]
const cid = CID.decode(myCid)
console.log(cid)
try {
// console.log(`myCid=${myCid} cid=${cid.toString()}`)
// const res = await ipfs.block.stat(cid, { withLocal: true, offline: true })
const res = await ipfs.block.stat(cid, { offline: true })
console.log(`myCid=${cid}, res=${JSON.stringify(res, null, 2)}`)
// break
} catch (error) {
file.isLocal = false
console.error(`Error fetching stats for cid=${cid}:`, error)
// break
}
}
files.push(file)
}The UI looks like this:
PS: Try the Project Apollo Archives and XKCD archives |
I wanted to check whether the browser is offline or online in JavaScript by using the const [isOnline, setIsOnline] = useState(navigator.onLine)
const CHECK_INTERNET_STATUS = 60 * 1000
function isGoogleOnline () {
return new Promise((resolve) => {
const img = document.createElement('img')
img.onerror = () => resolve(false)
img.onload = () => resolve(true)
img.src = 'https://www.google.com/favicon.ico?_=' + new Date().getTime()
})
}
// Function to create a delay
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms))
// Function to check status and log the result
async function checkInternetStatus () {
let onlineStatus = navigator.onLine
if (onlineStatus) {
console.log('You seems online! Checking Google...')
// Wait for 3 seconds
await delay(3000)
const googleOnline = await isGoogleOnline()
console.log('Google is', googleOnline ? 'online' : 'offline')
onlineStatus = googleOnline
} else {
console.log('No internet connection.')
}
setIsOnline(onlineStatus)
}
// Use useEffect to run the check once on component mount and set up an interval
useEffect(() => {
// Event listeners for online and offline status
const handleOnline = () => checkInternetStatus()
const handleOffline = () => setIsOnline(false)
window.addEventListener('online', handleOnline)
window.addEventListener('offline', handleOffline)
// Check status immediately on page load
checkInternetStatus()
// Set up the interval to check status every 15 seconds
const intervalId = setInterval(checkInternetStatus, CHECK_INTERNET_STATUS)
// Clean up the interval and event listeners when the component unmounts
return () => {
clearInterval(intervalId)
window.removeEventListener('online', handleOnline)
window.removeEventListener('offline', handleOffline)
}
}, []) // Empty dependency array to run only once on mountIf we're offline I'll apply this class to shade/unclick the file view .greyed-out {
color: grey;
pointer-events: none; /* This will prevent clicking */
opacity: 0.5;
}Should I choose only to use Thoughts? |
|
hey @acul71, thanks for the ping. We shouldn't include a dependency on a service like google.. I think a simple check for a 'hello world' CID should be sufficient for a backup online check? such as a fetch for If we query the local kubo node it could resolve immediately (due to being an inline CID) even if we're not actually online, so i'm thinking maybe a light fetch request to trustless-gateway.link? 
const url = 'https://trustless-gateway.link/ipfs/bafkqae2xmvwgg33nmuqhi3zajfiemuzahiwss?format=raw';
async function headRequest(url) {
try {
const response = await fetch(url, {
method: 'HEAD',
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
// Log status code
console.log('Status Code:', response.status);
} catch (error) {
console.error('Error making HEAD request:', error);
}
}
headRequest(url); |
I'm planning to modify doFilesFetch that handles the procedure. /**
* Adds file under the `src` path to the given `root` path. On completion will
* trigger `doFilesFetch` to update the state.
* @param {string} root
* @param {string} src
* @param {string} name
*/
doFilesAddPath: (root, src, name = '', downloadLocally = false) => perform(ACTIONS.ADD_BY_PATH, async (ipfs, { store }) => {
ensureMFS(store)
const path = realMfsPath(src)
const cid = /** @type {string} */(path.split('/').pop())
if (!name) {
name = cid
}
const dst = realMfsPath(join(root, name))
const srcPath = src.startsWith('/') ? src : `/ipfs/${cid}`
try {
console.log(`Adding ipfs.files.cp(${srcPath}, ${dst}) downloadLocally=${downloadLocally}`)
const cid = await ipfs.files.cp(srcPath, dst)
console.log(`Adding ipfs.files.cp(${srcPath}, ${dst}) = ${cid}`)
// ipfs refs --recursive --help
if (downloadLocally === true) {
for await (const ref of ipfs.refs(srcPath, { recursive: true })) {
if (ref.err) {
console.error('refs: ', ref.err)
} else {
console.log('refs: ', ref)
// output: "QmHash"
}
}
}
return cid
} finally {
await store.doFilesFetch()
}
}), |
.......... lidel (from Slack) React 4:59 ############################## @lidel @SgtPooki time_X 43 files 0.152 seconds
time_Apollo_albums.txt 105 files 0.422 seconds
time_XKCD_Archive.txt ~1800 files 1.429 secondsMaybe what I can do is submit this solution in the PR and open an issue or feat request in kubo, to let this info (local or not) come back with ipfs.files.stat. /**
* @param {IPFSService} ipfs
* @param {CID} cid
* @param {Object} options
* @param {string} options.path
* @param {boolean} [options.isRoot]
* @param {import('./utils').Sorting} options.sorting
*/
const dirStats = async (ipfs, cid, { path, isRoot, sorting }) => {
const entries = await all(ipfs.ls(cid)) || []
// Workarounds regression in IPFS HTTP Client that causes
// ls on empty dir to return list with that dir only.
// @see https://github.com/ipfs/js-ipfs/issues/3566
const res = (entries.length === 1 && entries[0].cid.toString() === cid.toString())
? []
: entries
const files = []
// precaution: there was a historical performance issue when too many dirs were present
let dirCount = 0
// let fileCount = 0
let totDuration = 0
for (const f of res) {
const absPath = join(path, f.name)
let file = null
if (dirCount < 1000 && (f.type === 'directory' || f.type === 'dir')) {
dirCount += 1
file = fileFromStats({ ...await stat(ipfs, f.cid), path: absPath })
} else {
file = fileFromStats({ ...f, path: absPath })
}
// Start timing
const startTime = performance.now()
// If dirCount > 300 stop checking and mark isLocal as undefined
if (dirCount > 300) {
// if (fileCount > 300) {
file.isLocal = undefined
} else {
// fileCount += 1
// Check if file is local
// TODO: See if # ipfs ls --size=false --resolve-type=false CID is faster then ipfs.dag.get CID
const refs = await ipfs.dag.get(f.cid)
// Extracting all CIDs
// const refsCids = refs.value.Links.map(link => link.Hash['/'])
const firstRefCid = refs.value.Links.length > 0 ? refs.value.Links[0].Hash['/'] : ''
// console.log(refsCids)
// console.log(`refs=${JSON.stringify(refs, null, 2)}`)
// if (refsCids.length === 0) {
if (firstRefCid === '') {
file.isLocal = true
} else {
file.isLocal = true
// for (const myCid of refsCids) {
// const myCid = refsCids[0]
// const cid = CID.decode(myCid)
const cid = CID.decode(firstRefCid)
// console.log(cid) // debug lucas
try {
// console.log(`myCid=${myCid} cid=${cid.toString()}`)
// const res = await ipfs.block.stat(cid, { offline: true })
await ipfs.block.stat(cid, { offline: true })
// console.log(`myCid=${cid}, res=${JSON.stringify(res, null, 2)}`)
// break
} catch (error) {
file.isLocal = false
console.error(`Error fetching stats for cid=${cid}:`, error)
// break
}
}
}
// End timing and log the duration
const endTime = performance.now()
// const duration = (endTime - startTime) / 1000 // Convert to seconds
const duration = (endTime - startTime)
totDuration += duration
files.push(file)
}
console.log(`Time taken to check if file is local: ${totDuration / 1000} secs`)
let parent = null
if (!isRoot) {
const parentPath = dirname(path)
const parentInfo = infoFromPath(parentPath, false)
if (parentInfo && (parentInfo.isMfs || !parentInfo.isRoot)) {
const realPath = parentInfo.realPath
if (realPath && realPath.startsWith('/ipns')) {
parentInfo.realPath = await last(ipfs.name.resolve(parentInfo.realPath))
}
parent = fileFromStats({
...await stat(ipfs, parentInfo.realPath),
path: parentInfo.path,
name: '..',
isParent: true
})
}
}
return {
path,
fetched: Date.now(),
type: 'directory',
cid,
upper: parent,
content: sortFiles(files, sorting)
}
} |
|
What is the exact meaning of files, pins, blocks, local repo ? Right know It's like this: I assume that:
About the expandable thingy (▶) |
I believe we want to add the expandable section so we can provide more information. It may not look great when expanded though, so we might need to pivot. However, let's start there |
@acul71 the issue #1248 you linked is from 2019 and is how old UI looked like. It was simplified since then (removed noise like pins and blocks). I wrote down my understanding of the current state of that header (and the sources of values) in #1248 (comment). |
This task aims to resolve all of the issues in the
Related itemssection below. The tasklist clarifies how we plan to accomplish this.Tasks
Some details about a method forward for this at ipfs/ipfs-desktop#2492 (comment)
Related items
The text was updated successfully, but these errors were encountered: