Use Redux for internal state plus add prototype for data hot reloading …

…fixes gatsbyjs#777 (gatsbyjs#788)

* Use DataTree instead of Abstract Syntax Tree

The idea for how to do things was taken from compilers
but since we're not actually parsing syntax into a tree, using
AST terminology is confusing. DataTree seems like a more sensible name.

* Add a watch command for all packages and update contributing instructions

* Add trailing slash when creating script tags for prefixed sites

* Add redux, convert all globals except pages to use it

* Add more tests + convert node api signature to (args, pluginOptions) to make same as browser/ssr api signature

* Use Redux for managing data nodes + initial rough implemenation of hot reloading

* Move raw action dispatching to action creator

* Add nodes tests

* Update windows docs

* Update CONTRIBUTING.md

* Somewhat reliable draining

* Upgrade gatsbygram for PR

* Up sites to latest canary

package.json

@@ -17,6 +17,7 @@
     "babel-cli": "^6.18.0",
     "babel-preset-es2015": "^6.18.0",
     "bluebird": "^3.4.6",
+    "chokidar": "^1.6.1",
     "fs-extra": "^1.0.0",
     "glob": "^7.1.1",
     "graphql": "^0.8.2",

src/gatsby-node.js

@@ -8,90 +8,78 @@ const fs = require("fs")
 const prettyBytes = require("pretty-bytes")
 const u = require("unist-builder")
 const slash = require("slash")
+const _ = require("lodash")
+const chokidar = require("chokidar")
 
-exports.sourceNodes = ({ args, pluginOptions }) =>
-  new Promise((resolve, reject) => {
-    console.time(`glob`)
-    recursive(
-      pluginOptions.path,
-      [
-        `.*.un~`,
-        `.gitignore`,
-        `.npmignore`,
-        `.babelrc`,
-        `yarn.lock`,
-        `../**/dist/**`,
-      ],
-      (err, files) => {
-        console.timeEnd(`glob`)
-        console.log(`total files`, files.length)
-
-        // Parse files
-        const mappedFiles = files.map(file => {
-          const slashed = slash(file)
-          return {
-            ...path.parse(slashed),
-            absolutePath: slashed,
-          }
+function readFile(file, pluginOptions, cb) {
+  const slashed = slash(file)
+  const slashedFile = {
+    ...path.parse(slashed),
+    absolutePath: slashed,
+  }
+  md5File(slashedFile.absolutePath, (md5Err, hash) => {
+    fs.stat(slashedFile.absolutePath, (statErr, stats) => {
+      // Stringify date objects.
+      const newFile = JSON.parse(
+        JSON.stringify({
+          type: `File`,
+          id: slashedFile.absolutePath,
+          sourceName: pluginOptions.name,
+          children: [],
+          relativePath: slash(
+            path.posix.relative(pluginOptions.path, slashedFile.absolutePath)
+          ),
+          extension: slashedFile.ext.slice(1).toLowerCase(),
+          size: stats.size,
+          prettySize: prettyBytes(stats.size),
+          modifiedTime: stats.mtime,
+          accessTime: stats.atime,
+          changeTime: stats.ctime,
+          birthTime: stats.birthtime,
+          hash,
+          ...slashedFile,
+          ...stats,
         })
+      )
+      cb(null, newFile)
+    })
+  })
+}
 
-        console.time(`hash files`)
-        // Get hash for each file.
-        mapLimit(
-          mappedFiles,
-          15,
-          (file, cb) => md5File(file.absolutePath, cb),
-          (e, hashes) => {
-            hashes.forEach((hash, index) => {
-              mappedFiles[index].hash = hash
-            })
-            console.timeEnd(`hash files`)
-
-            console.time(`stat files`)
-            mapLimit(
-              mappedFiles,
-              15,
-              (file, cb) => fs.stat(file.absolutePath, cb),
-              (er, stats) => {
-                stats.forEach((stat, index) => {
-                  mappedFiles[index] = {
-                    ...stat,
-                    ...mappedFiles[index],
-                  }
-                })
-                console.timeEnd(`stat files`)
+exports.sourceNodes = ({ actionCreators }, pluginOptions) => {
+  const { createNode, updateSourcePluginStatus } = actionCreators
+  updateSourcePluginStatus({
+    plugin: `source-filesystem --- ${pluginOptions.name}`,
+    ready: false,
+  })
+  let testNode
+  const watcher = chokidar.watch(pluginOptions.path, {
+    ignored: [
+      `**/*.un~`,
+      `**/.gitignore`,
+      `**/.npmignore`,
+      `**/.babelrc`,
+      `**/yarn.lock`,
+      `**/node_modules`,
+      `../**/dist/**`,
+    ],
+  })
 
-                console.log(`total files`, mappedFiles.length)
-                console.time(`create filesystem ast`)
-                // Create Unist nodes
-                const ast = u(`rootDirectory`, {}, pluginOptions.path)
-                ast.children = []
-                mappedFiles.forEach(file => {
-                  ast.children.push({
-                    ...JSON.parse(JSON.stringify(file)), // Stringify date objects.
-                    type: `File`,
-                    id: file.absolutePath,
-                    sourceName: pluginOptions.name,
-                    children: [],
-                    relativePath: slash(
-                      path.relative(pluginOptions.path, file.absolutePath)
-                    ),
-                    extension: file.ext.slice(1).toLowerCase(),
-                    size: file.size,
-                    prettySize: prettyBytes(file.size),
-                    modifiedTime: file.mtime.toJSON(),
-                    accessTime: file.atime.toJSON(),
-                    changeTime: file.ctime.toJSON(),
-                    birthTime: file.birthtime.toJSON(),
-                    hash: file.hash,
-                  })
-                })
-                console.timeEnd(`create filesystem ast`)
-                return resolve(ast)
-              }
-            )
-          }
-        )
-      }
-    )
+  watcher.on(`add`, path => {
+    // console.log("Added file at", path);
+    readFile(path, pluginOptions, (err, file) => createNode(file))
+  })
+  watcher.on(`change`, path => {
+    console.log("changed file at", path)
+    readFile(path, pluginOptions, (err, file) => createNode(file))
   })
+  watcher.on(`ready`, () => {
+    updateSourcePluginStatus({
+      plugin: `source-filesystem --- ${pluginOptions.name}`,
+      ready: true,
+    })
+  })
+
+  // TODO add delete support.
+  return
+}