[1.0] gatsbygram gravatar (#792)

* Update scraper to deal with avatar picture

* Generate new gatsbygram data

* Use avatar url instead of an hard-code gravatar

* Remove `want help` section :)

* Remove unused `instagram-screen-scrape` package

* Deal with pagination

* Regen data

* Avoid infinity loop if no more data

* Run yarn format

* Hack for get larger images

* Update gatsbygram data

docs/blog/gatsbygram-case-study/index.md

@@ -583,9 +583,6 @@ resulting "Gatsbygram" sites:
 * https://iceland-gatsbygram.netlify.com
 * https://tinyhouses-gatsbygram.netlify.com
 
-**Help wanted:** scrape the user's profile picture and use that instead
-of my Gravatar image which is hard-coded at the moment.
-
 *With thanks to Sam Bhagwatt, Sunil Pai, Nolan Lawson, Nik Graf,
 Jeff Posnick, and Addy Osmani for their reviews.*
 

examples/gatsbygram/components/post-detail.js

@@ -1,6 +1,5 @@
 import React from "react"
 import HeartIcon from "react-icons/lib/fa/heart"
-import Gravatar from "react-gravatar"
 
 import presets from "../utils/presets"
 import typography, { rhythm, scale } from "../utils/typography"
@@ -11,7 +10,15 @@ class PostDetail extends React.Component {
   }
 
   render() {
-    const { image, likes, id, username, weeksAgo, text } = this.props.post
+    const {
+      image,
+      likes,
+      id,
+      username,
+      weeksAgo,
+      text,
+      avatar,
+    } = this.props.post
     const { big } = image.children[0]
 
     const UserBar = () => (
@@ -23,11 +30,12 @@ class PostDetail extends React.Component {
           },
         }}
       >
-        <Gravatar
-          size={25}
-          email="mathews.kyle@gmail.com"
+        <img
+          src={avatar}
+          alt={username}
           css={{
             borderRadius: `100%`,
+            height: 25,
             float: `left`,
             margin: 0,
             marginRight: rhythm(1 / 2),

examples/gatsbygram/package.json

@@ -17,7 +17,6 @@
     "gatsby-source-filesystem": "1.0.0-alpha12-alpha.d7d6ef64",
     "gatsby-typegen-filesystem": "1.0.0-alpha12-alpha.d7d6ef64",
     "gatsby-typegen-sharp": "1.0.0-alpha12-alpha.d7d6ef64",
-    "instagram-screen-scrape": "^2.0.0",
     "lodash": "^4.16.4",
     "mkdirp": "^0.5.1",
     "mousetrap": "^1.6.0",

examples/gatsbygram/pages/index.js

@@ -1,6 +1,5 @@
 import React from "react"
 import chunk from "lodash/chunk"
-import Gravatar from "react-gravatar"
 
 import { rhythm, scale } from "../utils/typography"
 import presets from "../utils/presets"
@@ -81,9 +80,9 @@ class Index extends React.Component {
               flexShrink: 0,
             }}
           >
-            <Gravatar
-              size={151.2}
-              email="mathews.kyle@gmail.com"
+            <img
+              src={this.props.data.allPosts.edges[0].node.avatar}
+              alt={this.props.data.allPosts.edges[0].node.username}
               css={{
                 display: `block`,
                 margin: `0 auto`,
@@ -214,6 +213,7 @@ query allImages {
     edges {
       node {
         username
+        avatar
         likes
         id
         text

examples/gatsbygram/pages/template-post-page.js

@@ -25,6 +25,7 @@ export const pageQuery = `
     posts(id: { eq: $id }) {
       # Specify the fields from the post we need.
       username
+      avatar
       likes
       id
       text

examples/gatsbygram/scrape.js

@@ -1,8 +1,9 @@
-const { InstagramPosts } = require("instagram-screen-scrape")
-const url = require("url")
 const fs = require("fs")
-const ProgressBar = require("progress")
+const request = require("request")
 const mkdirp = require("mkdirp")
+const ProgressBar = require("progress")
+const { get } = require("lodash")
+const download = require("./utils/download-file")
 
 const username = process.argv[2]
 
@@ -18,13 +19,8 @@ node scrape.js INSTAGRAM_USERNAME
   process.exit()
 }
 
-const download = require("./utils/download-file")
-
-// Create the images directory
-mkdirp.sync(`./data/images`)
-
-// Create the stream
-const streamOfPosts = new InstagramPosts({ username })
+// Convert timestamp to ISO 8601.
+const toISO8601 = timestamp => new Date(timestamp * 1000).toJSON()
 
 // Create the progress bar
 const bar = new ProgressBar(
@@ -35,46 +31,59 @@ const bar = new ProgressBar(
   }
 )
 
-const posts = []
-streamOfPosts.on(`data`, post => {
-  // Ignore video files.
-  if (post && post.media && post.media.slice(-3) === `mp4`) {
-    return
-  }
+// Create the images directory
+mkdirp.sync(`./data/images`)
 
-  // Only download the first 100 posts.
-  if (posts.length >= 100) {
-    return
-  }
+let posts = []
 
-  // Up total
-  bar.total += 1
+// Write json
+const saveJSON = _ =>
+  fs.writeFileSync(`./data/posts.json`, JSON.stringify(posts, "", 2))
 
-  // Convert timestamp to ISO 8601.
-  post.time = new Date(post.time * 1000).toJSON()
+const getPosts = maxId => {
+  let url = `https://www.instagram.com/${username}/media`
+  if (maxId) url += `?max_id=${maxId}`
 
-  const parsedUrl = url.parse(post.media)
-  const splitPathname = parsedUrl.pathname.split(`/`)
-  const newPathname = []
-    .concat(splitPathname[0], splitPathname[1], splitPathname.slice(-1))
-    .join(`/`)
-  const newUrl = `${parsedUrl.protocol}//${parsedUrl.host}${newPathname}`
+  request(url, { encoding: `utf8` }, (err, res, body) => {
+    if (err) console.log(`error: ${err}`)
+    body = JSON.parse(body)
+    // Parse posts
+    let lastId
+    body.items
+      .filter(item => item.type === `image`)
+      .map(item => {
+        // Parse item to a simple object
+        return {
+          id: get(item, `id`),
+          code: get(item, `code`),
+          username: get(item, `user.username`),
+          avatar: get(item, `user.profile_picture`),
+          time: toISO8601(get(item, `created_time`)),
+          type: get(item, `type`),
+          likes: get(item, `likes.count`),
+          comment: get(item, `comments.count`),
+          text: get(item, `caption.text`),
+          media: get(item, `images.standard_resolution.url`, ``).replace(`/s640x640`, ``),
+          image: `images/${item.code}.jpg`,
+        }
+      })
+      .forEach(item => {
+        if (posts.length >= 100) return
 
-  // Delay downloading a bit so our scrapping can catchup so our progressbar
-  // proceeds smoothly.
-  setTimeout(
-    () => {
-      download(newUrl, `./data/images/${post.id}.jpg`, () => bar.tick())
-    },
-    1000
-  )
+        // Download image locally and update progress bar
+        bar.total++
+        download(item.media, `./data/images/${item.code}.jpg`, _ => bar.tick())
+
+        // Add item to posts
+        posts.push(item)
 
-  // In our json representation of the post, link to the downloaded image.
-  // This will be used in our Gatsby code to access the image.
-  post.image = `images/${post.id}.jpg`
-  posts.push(post)
-})
+        // Save lastId for next request
+        lastId = item.id
+      })
+
+    if (posts.length < 100 && get(body, `more_available`)) getPosts(lastId)
+    else saveJSON()
+  })
+}
 
-streamOfPosts.on(`end`, () => {
-  fs.writeFileSync(`./data/posts.json`, JSON.stringify(posts))
-})
+getPosts()

examples/gatsbygram/yarn.lock

@@ -2,13 +2,6 @@
 # yarn lockfile v1
 
 
-JSONStream@^0.10.0:
-  version "0.10.0"
-  resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-0.10.0.tgz#74349d0d89522b71f30f0a03ff9bd20ca6f12ac0"
-  dependencies:
-    jsonparse "0.0.5"
-    through ">=2.2.7 <3"
-
 abbrev@1:
   version "1.0.9"
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135"
@@ -3283,11 +3276,11 @@ got@^6.7.1:
     unzip-response "^2.0.1"
     url-parse-lax "^1.0.0"
 
-graceful-fs@^4.0.0, graceful-fs@^4.1.11, graceful-fs@^4.1.3, graceful-fs@^4.1.4, graceful-fs@^4.1.6:
+graceful-fs@^4.0.0, graceful-fs@^4.1.11:
   version "4.1.11"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658"
 
-graceful-fs@^4.1.2:
+graceful-fs@^4.1.2, graceful-fs@^4.1.3, graceful-fs@^4.1.4, graceful-fs@^4.1.6:
   version "4.1.9"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.9.tgz#baacba37d19d11f9d146d3578bc99958c3787e29"
 
@@ -3695,16 +3688,6 @@ ini@^1.3.4, ini@~1.3.0:
   version "1.3.4"
   resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.4.tgz#0537cb79daf59b59a1a517dff706c86ec039162e"
 
-instagram-screen-scrape@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/instagram-screen-scrape/-/instagram-screen-scrape-2.0.0.tgz#d1c31fc9d716829750e64524a52b8ed0cd2aaf84"
-  dependencies:
-    JSONStream "^0.10.0"
-    argparse "^1.0.7"
-    readable-stream "^2.0.4"
-    request "^2.65.0"
-    tough-cookie "^2.2.1"
-
 interpret@^0.6.4:
   version "0.6.6"
   resolved "https://registry.yarnpkg.com/interpret/-/interpret-0.6.6.tgz#fecd7a18e7ce5ca6abfb953e1f86213a49f1625b"
@@ -4148,10 +4131,6 @@ jsonify@~0.0.0:
   version "0.0.0"
   resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73"
 
-jsonparse@0.0.5:
-  version "0.0.5"
-  resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-0.0.5.tgz#330542ad3f0a654665b778f3eb2d9a9fa507ac64"
-
 jsonpointer@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-4.0.0.tgz#6661e161d2fc445f19f98430231343722e1fcbd5"
@@ -4506,11 +4485,11 @@ lodash@4.11.1:
   version "4.11.1"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.11.1.tgz#a32106eb8e2ec8e82c241611414773c9df15f8bc"
 
-lodash@^4.1.0, lodash@^4.14.0, lodash@^4.16.2, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.6.1:
+lodash@^4.1.0, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.16.2, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.6.1:
   version "4.17.4"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
 
-lodash@^4.13.1, lodash@^4.16.4:
+lodash@^4.16.4:
   version "4.16.4"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.16.4.tgz#01ce306b9bad1319f2a5528674f88297aeb70127"
 
@@ -6241,9 +6220,9 @@ rechoir@^0.6.2:
   dependencies:
     resolve "^1.1.6"
 
-"recursive-readdir@git+https://github.com/KyleAMathews/recursive-readdir.git":
+"recursive-readdir@https://github.com/KyleAMathews/recursive-readdir.git":
   version "2.1.0"
-  resolved "git+https://github.com/KyleAMathews/recursive-readdir.git#fbfe835d2e75a977db2d1e794214aa26ecdb0671"
+  resolved "https://github.com/KyleAMathews/recursive-readdir.git#fbfe835d2e75a977db2d1e794214aa26ecdb0671"
   dependencies:
     minimatch "3.0.2"
 
@@ -6447,7 +6426,7 @@ replace-ext@1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.0.tgz#de63128373fcbf7c3ccfa4de5a480c45a67958eb"
 
-request@2.x, request@^2.58.0, request@^2.65.0, request@^2.67.0, request@^2.74.0, request@^2.79.0:
+request@2.x, request@^2.58.0, request@^2.67.0, request@^2.74.0, request@^2.79.0:
   version "2.79.0"
   resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
   dependencies:
@@ -7179,10 +7158,6 @@ through2@^2.0.0, through2@~2.0.0:
     readable-stream "^2.1.5"
     xtend "~4.0.1"
 
-"through@>=2.2.7 <3":
-  version "2.3.8"
-  resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
-
 time-stamp@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/time-stamp/-/time-stamp-1.0.1.tgz#9f4bd23559c9365966f3302dbba2b07c6b99b151"
@@ -7239,7 +7214,7 @@ topo@2.x.x:
   dependencies:
     hoek "4.x.x"
 
-tough-cookie@^2.2.1, tough-cookie@~2.3.0:
+tough-cookie@~2.3.0:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.1.tgz#99c77dfbb7d804249e8a299d4cb0fd81fef083fd"