Merge pull request #5837 from ForumMagnum/user-agent-for-crosspost-gr…

…aphql

add user agent to xpost url fetch

package.json

@@ -143,7 +143,6 @@
     "compression": "^1.7.3",
     "connect-mongo": "^4.4.1",
     "cookie-parser": "^1.4.3",
-    "cross-fetch": "^3.1.5",
     "crypto-js": "^3.1.9-1",
     "dataloader": "^1.4.0",
     "deepmerge": "^1.2.0",
@@ -206,6 +205,7 @@
     "moment": "^2.29.4",
     "mongodb": "^3.6.3",
     "mz": "^2.7.0",
+    "node-fetch": "^2.6",
     "nodemailer": "^6.6.1",
     "papaparse": "^4.6.2",
     "passport": "^0.5.2",

packages/lesswrong/lib/apollo/links.ts

@@ -4,6 +4,8 @@ import { BatchHttpLink } from '@apollo/client/link/batch-http';
 import { onError } from '@apollo/client/link/error';
 import { isServer } from '../executionEnvironment';
 
+export const crosspostUserAgent = "ForumMagnum/2.1";
+
 /**
  * "Links" are Apollo's way of defining the source to read our data from, and they need to
  * be set up differently depending on whether we're rendering on the server or on the client,
@@ -22,14 +24,30 @@ export const createSchemaLink = (schema: GraphQLSchema, context: ResolverContext
 /**
  * Http link is used for client side rendering
  */
-export const createHttpLink = (baseUrl = '/') =>
-  new BatchHttpLink({
+export const createHttpLink = (baseUrl = '/') => {
+  // Type of window.fetch may differ slightly from type of the fetch used on server
+  let fetch: typeof window.fetch;
+  if (isServer) {
+    // We won't need to import fetch in node 18
+    const nodeFetch = require('node-fetch');
+    fetch = (url, options) => nodeFetch(url, {
+      ...options,
+      headers: {
+        ...options?.headers,
+        // user agent because LW bans bot agents
+        'User-Agent': crosspostUserAgent,
+      }
+    });
+  } else {
+    fetch = window.fetch;
+  }
+  return new BatchHttpLink({
     uri: baseUrl + 'graphql',
     credentials: baseUrl === '/' ? 'same-origin' : 'omit',
     batchMax: 50,
-    // TODO: This line can be removed once we upgrade to node v18
-    fetch: isServer ? require("cross-fetch") : window.fetch,
+    fetch,
   });
+}
 
 const locationsToStr = (locations: readonly SourceLocation[] = []) =>
   locations.map(({column, line}) => `line ${line}, col ${column}`).join(';');

packages/lesswrong/server/fmCrosspost/crosspost.ts

@@ -4,7 +4,8 @@ import { randomId } from "../../lib/random";
 import { Crosspost, UpdateCrosspostPayload, CrosspostPayload } from "./types";
 import { signToken } from "./tokens";
 import { apiRoutes, makeApiUrl } from "./routes";
-import { makeCrossSiteRequest, crosspostUserAgent } from "./resolvers";
+import { makeCrossSiteRequest } from "./resolvers";
+import { crosspostUserAgent } from "../../lib/apollo/links";
 
 export const performCrosspost = async <T extends Crosspost>(post: T): Promise<T> => {
   if (!post.fmCrosspost || !post.userId || post.draft) {

packages/lesswrong/server/fmCrosspost/resolvers.ts

@@ -6,8 +6,7 @@ import { ApiRoute, apiRoutes, makeApiUrl } from "./routes";
 import { signToken } from "./tokens";
 import type { Request } from "express";
 import fetch from "node-fetch";
-
-export const crosspostUserAgent = "ForumMagnum/2.1";
+import { crosspostUserAgent } from "../../lib/apollo/links";
 
 const getUserId = (req?: Request) => {
   const userId = req?.user?._id;

yarn.lock

@@ -6003,13 +6003,6 @@ cross-fetch@3.0.6, cross-fetch@^3.0.6:
   dependencies:
     node-fetch "2.6.1"
 
-cross-fetch@^3.1.5:
-  version "3.1.5"
-  resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f"
-  integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==
-  dependencies:
-    node-fetch "2.6.7"
-
 cross-spawn@^6.0.0:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz"
@@ -11673,13 +11666,6 @@ node-fetch@2.6.1, node-fetch@^2.1.2, node-fetch@^2.6.0, node-fetch@^2.6.1:
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz"
   integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
 
-node-fetch@2.6.7, node-fetch@^2.6.7:
-  version "2.6.7"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
-  integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
-  dependencies:
-    whatwg-url "^5.0.0"
-
 node-fetch@^1.0.1:
   version "1.7.3"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz"
@@ -11688,6 +11674,13 @@ node-fetch@^1.0.1:
     encoding "^0.1.11"
     is-stream "^1.0.1"
 
+node-fetch@^2.6, node-fetch@^2.6.7:
+  version "2.6.7"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
+  integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-int64@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz"