Pinning core, cli and http

Author: AdamStone

package.json

@@ -93,15 +93,17 @@
     "async": "^2.5.0",
     "bl": "^1.2.1",
     "boom": "^5.2.0",
-    "debug": "^3.0.1",
     "cids": "^0.5.1",
+    "debug": "^3.0.1",
     "file-type": "^6.1.0",
     "filesize": "^3.5.10",
+    "fnv1a": "^1.0.1",
     "fsm-event": "^2.1.0",
     "glob": "^7.1.2",
     "hapi": "^16.5.2",
     "hapi-set-header": "^1.0.2",
     "hoek": "^4.2.0",
+    "interface-datastore": "^0.3.0",
     "ipfs-api": "^14.3.5",
     "ipfs-bitswap": "~0.17.2",
     "ipfs-block": "~0.6.0",
@@ -126,6 +128,7 @@
     "libp2p-websockets": "~0.10.1",
     "lodash.flatmap": "^4.5.0",
     "lodash.get": "^4.4.2",
+    "lodash.set": "^4.3.0",
     "lodash.sortby": "^4.7.0",
     "lodash.values": "^4.3.0",
     "mafmt": "^3.0.1",
@@ -139,6 +142,7 @@
     "peer-id": "~0.10.1",
     "peer-info": "~0.11.0",
     "promisify-es6": "^1.0.3",
+    "protocol-buffers": "^3.2.1",
     "pull-file": "^1.0.0",
     "pull-paramap": "^1.2.2",
     "pull-pushable": "^2.1.1",
@@ -216,4 +220,4 @@
     "Łukasz Magiera <[email protected]>",
     "ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ <[email protected]>"
   ]
-}
+}

src/cli/commands/pin.js

@@ -0,0 +1,15 @@
+'use strict'
+
+module.exports = {
+  command: 'pin',
+
+  description: 'Pin and unpin objects to local storage.',
+
+  builder (yargs) {
+    return yargs
+      .commandDir('pin')
+  },
+
+  handler (argv) {
+  }
+}

src/cli/commands/pin/add.js

@@ -0,0 +1,28 @@
+'use strict'
+
+module.exports = {
+  command: 'add <ipfs-path>',
+
+  describe: 'Pins objects to local storage.',
+
+  builder: {
+    recursive: {
+      type: 'boolean',
+      alias: 'r',
+      default: true,
+      describe: 'Recursively pin the object linked to by the specified object(s).'
+    }
+  },
+
+  handler (argv) {
+    const paths = argv['ipfs-path'].split(' ')
+    const recursive = argv.recursive
+    const type = recursive ? 'recursive' : 'direct'
+    argv.ipfs.pin.add(paths[0], { recursive }, (err, results) => {
+      if (err) { throw err }
+      results.forEach((res) => {
+        console.log(`pinned ${res.hash} ${type}ly`)
+      })
+    })
+  }
+}

src/cli/commands/pin/ls.js

@@ -0,0 +1,43 @@
+'use strict'
+
+module.exports = {
+  command: 'ls',
+
+  describe: 'List objects pinned to local storage.',
+
+  builder: {
+    path: {
+      type: 'string',
+      describe: 'List pinned state of specific <ipfs-path>.'
+    },
+    type: {
+      type: 'string',
+      alias: 't',
+      default: 'all',
+      describe: ('The type of pinned keys to list. ' +
+                 'Can be "direct", "indirect", "recursive", or "all".')
+    },
+    quiet: {
+      type: 'boolean',
+      alias: 'q',
+      default: false,
+      describe: 'Write just hashes of objects.'
+    }
+  },
+
+  handler: (argv) => {
+    const paths = argv.path && argv.path.split(' ')
+    const type = argv.type
+    const quiet = argv.quiet
+    argv.ipfs.pin.ls(paths, { type }, (err, results) => {
+      if (err) { throw err }
+      results.forEach((res) => {
+        let line = res.hash
+        if (!quiet) {
+          line += ` ${res.type}`
+        }
+        console.log(line)
+      })
+    })
+  }
+}

src/cli/commands/pin/rm.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const utils = require('../../utils')
+const debug = require('debug')
+const log = debug('cli:pin')
+log.error = debug('cli:pin:error')
+
+module.exports = {
+  command: 'rm <ipfs-path>',
+
+  describe: 'Removes the pinned object from local storage.',
+
+  builder: {
+    recursive: {
+      type: 'boolean',
+      alias: 'r',
+      default: true,
+      describe: 'Recursively unpin the objects linked to by the specified object(s).'
+    }
+  },
+
+  handler: (argv) => {
+    const paths = argv['ipfs-path'].split(' ')
+    const recursive = argv.recursive
+    utils.getIPFS((err, ipfs) => {
+      if (err) { throw err }
+      ipfs.pin.rm(paths, { recursive }, (err, results) => {
+        if (err) { throw err }
+        results.forEach((res) => {
+          console.log(`unpinned ${res.hash}`)
+        })
+      })
+    })
+  }
+}

src/core/boot.js

@@ -28,6 +28,7 @@ module.exports = (self) => {
 
     series([
       (cb) => self._repo.open(cb),
+      (cb) => self.pin.load(cb),
       (cb) => self.preStart(cb),
       (cb) => {
         self.state.initialized()

src/core/components/files.js

@@ -74,6 +74,11 @@ module.exports = function files (self) {
           if (a.path > b.path) return -1
           return 0
         }),
+        pull.asyncMap((file, cb) => {
+          self.pin.add(file.hash, (err) => {
+            cb(err, file)
+          })
+        }),
         pull.collect(callback)
       )
     }),

src/core/components/index.js

@@ -16,6 +16,7 @@ exports.dag = require('./dag')
 exports.libp2p = require('./libp2p')
 exports.swarm = require('./swarm')
 exports.ping = require('./ping')
+exports.pin = require('./pin')
 exports.files = require('./files')
 exports.bitswap = require('./bitswap')
 exports.pubsub = require('./pubsub')

src/core/components/key-set.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const multihashes = require('multihashes')
+
+module.exports = function KeySet (keys) {
+  // Buffers with identical data are still different objects, so
+  // they need to be cast to strings to prevent duplicates in Sets
+  this.keys = {}
+  this.add = (key) => {
+    this.keys[multihashes.toB58String(key)] = key
+  }
+  this.delete = (key) => {
+    delete this.keys[multihashes.toB58String(key)]
+  }
+  this.clear = () => {
+    this.keys = {}
+  }
+  this.has = (key) => {
+    return (multihashes.toB58String(key) in this.keys)
+  }
+  this.toArray = () => {
+    return Object.keys(this.keys).map((hash) => {
+      return this.keys[hash]
+    })
+  }
+  this.toStringArray = () => {
+    return Object.keys(this.keys)
+  }
+  keys = keys || []
+  keys.forEach(this.add)
+}