feat: Implement unpin feature

md/usage.md

@@ -75,6 +75,7 @@ Options:
   -c, --cid           Pin this CID instead of uploading
   -q, --quiet         Only print the CID in the end   [boolean] [default: false]
   -H, --hidden        Add hidden (dot) files to IPFS  [boolean] [default: false]
+      --unpin-old     Unpin old requires a DNS linker [boolean] [default: false]
   -h, --help          Show help                                        [boolean]
 
 Examples:
@@ -104,6 +105,10 @@ Please keep in mind:
   sure you want them to be added, use the flag `-H, --hidden`.
 - All of the services are subject to their terms.
 
+## Unpinning
+It works by getting CID to unpin from dnslinker and then tries to unpin it from every
+upload and pin service. Currently, implemented only for CloudFlare and pinners that
+derive from IpfsNode (c4rex, DAppNode, Infura).
 
 ## Uploading and Pining
 
@@ -212,18 +217,18 @@ being updated.
     - `IPFS_DEPLOY_CLOUDFLARE__API_TOKEN=<scoped token>`
   - Configuration
     - `IPFS_DEPLOY_CLOUDFLARE__ZONE=<zone>`
-    - `IPFS_DEPLOY_CLOUDFLARE__RECORD=<record>`
+    - `IPFS_DEPLOY_CLOUDFLARE__WEB3_HOSTNAME=<web3 hostname>`
 
 #### Examples
 
 ```bash
 # Top level domain
 IPFS_DEPLOY_CLOUDFLARE__ZONE=example.com
-IPFS_DEPLOY_CLOUDFLARE__RECORD=_dnslink.example.com
+IPFS_DEPLOY_CLOUDFLARE__WEB3_HOSTNAME=example.com
 
 # Subdomain
 IPFS_DEPLOY_CLOUDFLARE__ZONE=example.com
-IPFS_DEPLOY_CLOUDFLARE__RECORD=_dnslink.mysubdomain.example.com
+IPFS_DEPLOY_CLOUDFLARE__WEB3_HOSTNAME=mysubdomain.example.com
 ```
 
 ### [AWS-Route53](https://aws.amazon.com/route53/)

package.json

@@ -71,11 +71,12 @@
     "byte-size": "^8.1.0",
     "chalk": "^4.1.1",
     "clipboardy": "^2.3.0",
-    "dnslink-cloudflare": "^3.0.0",
+    "dnslink-cloudflare": "banciur/dnslink-cloudflare#banciur/handle_web3",
     "dnslink-dnsimple": "^1.0.1",
     "dotenv": "^16.0.0",
     "dreamhost": "^1.0.5",
     "form-data": "^4.0.0",
+    "got": "^11.5.1",
     "ipfs-http-client": "^50.1.0",
     "it-all": "^1.0.6",
     "lodash.isempty": "^4.4.0",

src/cli.js

@@ -78,6 +78,11 @@ const argv = yargs
             describe: 'Add hidden (dot) files to IPFS',
             type: 'boolean',
             default: false
+          },
+          'unpin-old': {
+            describe: 'Unpin old requires a DNS linker',
+            type: 'boolean',
+            default: false
           }
         })
         .example(
@@ -116,6 +121,7 @@ const options = {
   copyUrl: !argv.C,
   openUrls: argv.open,
   hiddenFiles: argv.hidden,
+  unpinOld: argv.unpinOld,
 
   uploadServices: arrayFromString(argv.upload),
   pinningServices: arrayFromString(argv.pinner),
@@ -127,7 +133,7 @@ const options = {
       apiToken: argv.cloudflare && argv.cloudflare.apiToken,
       apiEmail: argv.cloudflare && argv.cloudflare.apiEmail,
       zone: argv.cloudflare && argv.cloudflare.zone,
-      record: argv.cloudflare && argv.cloudflare.record
+      web3Hostname: argv.cloudflare && argv.cloudflare.web3Hostname
     },
     route53: {
       accessKeyId: argv.route53 && argv.route53.accessKeyId,

src/deploy.js

@@ -185,6 +185,38 @@ async function checkDirAndCid (dir, cid, logger) {
   return { cid, dir }
 }
 
+/**
+ * @param {DNSLinker[]} dnsServices
+ * @param {PinningService[]} pinServices
+ * @param {Logger} logger
+ */
+async function unpin (dnsServices, pinServices, logger) {
+  /** @type {string[]} */
+  const linkedCids = []
+
+  for (const dnsProvider of dnsServices) {
+    logger.info(`Getting linked cid from ${dnsProvider.displayName}`)
+    const cid = await dnsProvider.getLinkedCid()
+    logger.info(`Got cid: ${cid}`)
+    linkedCids.push(cid)
+  }
+
+  if (linkedCids.some(v => v !== linkedCids[0])) {
+    throw new Error(`Found inconsistency in linked CIDs: ${linkedCids}`)
+  }
+
+  const cidToUnpin = linkedCids[0]
+  if (!cidToUnpin) {
+    logger.info('There is nothing to unpin')
+    return
+  }
+
+  for (const pinProvider of pinServices) {
+    logger.info(`Unpinning ${cidToUnpin} from ${pinProvider.displayName}`)
+    await pinProvider.unpinCid(cidToUnpin, logger)
+  }
+}
+
 /**
  * @param {DeployOptions} options
  * @returns {Promise<string>}
@@ -197,6 +229,7 @@ async function deploy ({
   copyUrl = false,
   openUrls = false,
   hiddenFiles = false,
+  unpinOld = false,
 
   uploadServices: uploadServicesIds = [],
   pinningServices: pinningServicesIds = [],
@@ -242,9 +275,17 @@ async function deploy ({
   logger.info('⚙️   Validating DNS providers configurations…')
   const dnsProviders = dnsProvidersIds.map(name => {
     const DNSLinker = dnsLinkersMap.get(name)
+    // logger.info(dnsProvidersCredentials[name])
     return new DNSLinker(dnsProvidersCredentials[name])
   })
 
+  if (unpinOld) {
+    if (dnsProviders.length === 0) {
+      throw new Error('If you want to unpin you must provide dns provider')
+    }
+    await unpin(dnsProviders, uploadServices.concat(pinningServices), logger)
+  }
+
   const pinnedCids = /** @type {string[]} */([])
   const gatewayUrls = /** @type {string[]} */([])
 

src/dnslinkers/cloudflare.js

@@ -3,6 +3,7 @@
 // @ts-ignore
 const dnslink = require('dnslink-cloudflare')
 const isEmpty = require('lodash.isempty')
+const getLinkedCid = require('../lib/cloudflareCid')
 
 /**
  * @typedef {import('./types').DNSRecord} DNSRecord
@@ -13,13 +14,13 @@ class Cloudflare {
   /**
    * @param {CloudflareOptions} options
    */
-  constructor ({ apiEmail, apiKey, apiToken, zone, record }) {
+  constructor ({ apiEmail, apiKey, apiToken, zone, web3Hostname }) {
     if ([apiKey, apiEmail, apiToken].every(isEmpty)) {
       throw new Error('apiEmail and apiKey or apiToken are required for Cloudflare')
     }
 
-    if ([zone, record].some(isEmpty)) {
-      throw new Error('zone and record are required for CloudFlare')
+    if ([zone, web3Hostname].some(isEmpty)) {
+      throw new Error('zone and web3 hostname are required for CloudFlare')
     }
 
     if (isEmpty(apiKey)) {
@@ -31,7 +32,7 @@ class Cloudflare {
       }
     }
 
-    this.opts = { record, zone }
+    this.opts = { web3Hostname, zone }
   }
 
   /**
@@ -47,11 +48,18 @@ class Cloudflare {
     const content = await dnslink(this.api, opts)
 
     return {
-      record: opts.record,
+      record: opts.web3Hostname,
       value: content
     }
   }
 
+  /**
+   * @returns {Promise<string|null>}
+   */
+  async getLinkedCid () {
+    return getLinkedCid(this.api, this.opts)
+  }
+
   static get displayName () {
     return 'Cloudflare'
   }

src/dnslinkers/dnsimple.js

@@ -51,6 +51,13 @@ class DNSimple {
     }
   }
 
+  /**
+   * @returns {Promise<string>}
+   */
+  async getLinkedCid () {
+    throw new Error('getLinkedCid not implemented in DNSimple')
+  }
+
   static get displayName () {
     return 'DNSimple'
   }

src/dnslinkers/dreamhost.js

@@ -68,6 +68,10 @@ class DreamHost {
     })
   }
 
+  async getLinkedCid () {
+    throw new Error('getLinkedCid not implemented in DreamHost')
+  }
+
   static get displayName () {
     return 'DreamHost'
   }

src/dnslinkers/route53.js

@@ -68,6 +68,10 @@ class Route53 {
     }
   }
 
+  async getLinkedCid () {
+    throw new Error('getLinkedCid not implemented in Route53')
+  }
+
   static get displayName () {
     return 'Route53'
   }

src/dnslinkers/types.ts

@@ -5,6 +5,7 @@ export interface DNSRecord {
 
 export interface DNSLinker {
   link: (cid: string) => Promise<DNSRecord>
+  getLinkedCid: () => Promise<string>
   displayName: string
 }
 
@@ -13,7 +14,7 @@ export interface CloudflareOptions {
   apiKey?: string
   apiToken?: string
   zone: string
-  record: string
+  web3Hostname: string
 }
 
 export interface DNSimpleOptions {

src/lib/cloudflareCid.js

@@ -0,0 +1,66 @@
+// @ts-nocheck
+'use strict'
+// This is copy of dnslink-cloudflare with removed update and added getLinkedCid.
+const got = require('got')
+
+async function getZoneId (api, name) {
+  let res
+
+  for (let i = 1; (res = await api(`zones?page=${i}`)) && res.body.result_info.total_pages >= i; i++) {
+    for (const zone of res.body.result) {
+      if (zone.name === name) {
+        return zone.id
+      }
+    }
+  }
+
+  throw new Error(`zone ${name} couldn't be found`)
+}
+
+async function getRecord (api, id, name) {
+  let res
+
+  for (let i = 1; (res = await api(`zones/${id}/dns_records?type=TXT&page=${i}`)) && res.body.result_info.total_pages >= i; i++) {
+    for (const record of res.body.result) {
+      if (record.name.includes(name) && record.content.startsWith('dnslink=')) {
+        return record
+      }
+    }
+  }
+
+  return null
+}
+
+function getClient (apiOpts) {
+  const opts = {
+    prefixUrl: 'https://api.cloudflare.com/client/v4',
+    responseType: 'json'
+  }
+
+  if (apiOpts.token) {
+    opts.headers = {
+      Authorization: `Bearer ${apiOpts.token}`
+    }
+  } else {
+    opts.headers = {
+      'X-Auth-Email': apiOpts.email,
+      'X-Auth-Key': apiOpts.key
+    }
+  }
+
+  return got.extend(opts)
+}
+
+async function getLinkedCid (apiOpts, { zone, web3Hostname }) {
+  const api = getClient(apiOpts)
+  const id = await getZoneId(api, zone)
+  const zoneRecord = await getRecord(api, id, web3Hostname)
+  if (zoneRecord) {
+    // assuming content 'dnslink=/ipfs/QmZ4tDuvesekSs4qM5ZBKpXiZGun7S2CYtEZRB3DYXkjGx'
+    return zoneRecord.content.slice(zoneRecord.content.lastIndexOf('/') + 1)
+  } else {
+    return null
+  }
+}
+
+module.exports = getLinkedCid

src/pinners/ipfs-cluster.js

@@ -8,6 +8,7 @@ const { getDirFormData } = require('./utils')
 /**
  * @typedef {import('./types').IPFSClusterOptions} IPFSClusterOptions
  * @typedef {import('./types').PinDirOptions} PinDirOptions
+ * @typedef {import('../types').Logger} Logger
  */
 
 class IpfsCluster {
@@ -73,6 +74,14 @@ class IpfsCluster {
       })
   }
 
+  /**
+   * @param {string} cid
+   * @param {Logger} logger
+   */
+  async unpinCid (cid, logger) {
+    throw new Error('unpinCid not implemented in IpfsCluster')
+  }
+
   /**
    * @param {string} cid
    * @returns string

src/pinners/ipfs-node.js

@@ -7,6 +7,7 @@ const path = require('path')
 /**
  * @typedef {import('ipfs-http-client').Options} IpfsOptions
  * @typedef {import('./types').PinDirOptions} PinDirOptions
+ * @typedef {import('../types').Logger} Logger
  */
 
 class IpfsNode {
@@ -43,6 +44,22 @@ class IpfsNode {
     await this.ipfs.pin.add(cid)
   }
 
+  /**
+   * @param {string} cid
+   * @param {Logger} logger
+   */
+  async unpinCid (cid, logger) {
+    try {
+      await this.ipfs.pin.rm(cid)
+    } catch (e) {
+      if (e.name === 'HTTPError' && e.message === 'not pinned or pinned indirectly') {
+        logger.info(`${cid} not pinned to ${this.displayName}, moving forward`)
+      } else {
+        throw (e)
+      }
+    }
+  }
+
   /**
    * @param {string} cid
    * @returns string

src/pinners/pinata.js

@@ -7,6 +7,7 @@ const { getDirFormData } = require('./utils')
 /**
  * @typedef {import('./types').PinataOptions} PinataOptions
  * @typedef {import('./types').PinDirOptions} PinDirOptions
+ * @typedef {import('../types').Logger} Logger
  */
 
 const MAX_RETRIES = 3
@@ -91,6 +92,14 @@ class Pinata {
     await axios.post(PIN_HASH_URL, body, config)
   }
 
+  /**
+   * @param {string} cid
+   * @param {Logger} logger
+   */
+  async unpinCid (cid, logger) {
+    throw new Error('unpinCid not implemented in Pinata')
+  }
+
   /**
    * @param {string} cid
    * @returns string