[WIP] Bolt monorepo for NPM migration

cli/src/commands/__tests__/__install-fixtures__/installNpmLibDef/fakeCacheRepo/experimental/definitions/@babel/code-frame/code-frame_v7.x.x/flow_v0.30.x-v0.83.x/index.js

@@ -0,0 +1,64 @@
+declare module '@babel/code-frame' {
+  declare export type SourceLocation = {
+    start: {|
+      line: number,
+      column?: number,
+    |},
+    end?: {|
+      line: number,
+      column?: number,
+    |},
+  };
+  declare export function codeFrameColumns(
+    rawLines: string,
+    location: SourceLocation,
+    options?: BabelCodeFrameOptions
+  ): string;
+
+  declare export type BabelCodeFrameOptions = {
+    /**
+     * Syntax highlight the code as JavaScript for terminals. default: false
+     */
+    highlightCode?: boolean,
+
+    /**
+     * The number of lines to show above the error. default: 2
+     */
+    linesAbove?: number,
+
+    /**
+     * The number of lines to show below the error. default: 3
+     */
+    linesBelow?: number,
+
+    /**
+     * Forcibly syntax highlight the code as JavaScript (for non-terminals);
+     * overrides highlightCode.
+     * default: false
+     */
+    forceColor?: boolean,
+
+    /**
+     * Pass in a string to be displayed inline (if possible) next to the
+     * highlighted location in the code. If it can't be positioned inline,
+     * it will be placed above the code frame.
+     * default: nothing
+     */
+    message?: string,
+  }
+
+  /**
+   * Generate errors that contain a code frame that point to source locations.
+   * @param rawLines Raw lines to frame
+   * @param lineNumber Line number (1 indexed)
+   * @param colNumber Column number
+   * @param options Additional options
+   * @returns Framed code
+   */
+  declare export default function codeFrame(
+    rawLines: string,
+    lineNumber: number,
+    colNumber: number,
+    options?: BabelCodeFrameOptions
+  ): string;
+}

cli/src/commands/__tests__/__install-fixtures__/installNpmLibDef/fakeCacheRepo/experimental/definitions/@babel/code-frame/code-frame_v7.x.x/flow_v0.30.x-v0.83.x/tests/test_code-frame_v7.x.x.js

@@ -0,0 +1,24 @@
+// @flow
+
+import {describe, it} from 'flow-typed-test';
+import codeFrame, {codeFrameColumns} from '@babel/code-frame';
+
+const code = `
+  const number = 1;
+  var string = 'foo';
+  function print(name: string) {
+    console.log(string + name);
+  }
+`;
+
+it('codeFrame', () => {
+  codeFrame(code, 5, 22);
+  codeFrame(code, 5, 22, {forceColor: true});
+  codeFrame(code, 2, 2, {highlightCode: true});
+});
+
+it('codeFrameColumns', () => {
+  codeFrameColumns(code, {start: {line: 5, column: 22}});
+  codeFrameColumns(code, {start: {line: 5, column: 22}}, {forceColor: true});
+  codeFrameColumns(code, {start: {line: 2, column: 2}}, {highlightCode: true});
+});

cli/src/commands/__tests__/__install-fixtures__/installNpmLibDef/fakeCacheRepo/experimental/definitions/@babel/code-frame/code-frame_v7.x.x/flow_v0.84.x-/index.js

@@ -0,0 +1,66 @@
+declare module '@babel/code-frame' {
+  declare export type SourceLocation = {
+    start: {|
+      line: number,
+      column?: number,
+    |},
+    end?: {|
+      line: number,
+      column?: number,
+    |},
+    ...
+  };
+  declare export function codeFrameColumns(
+    rawLines: string,
+    location: SourceLocation,
+    options?: BabelCodeFrameOptions
+  ): string;
+
+  declare export type BabelCodeFrameOptions = {
+    /**
+     * Syntax highlight the code as JavaScript for terminals. default: false
+     */
+    highlightCode?: boolean,
+
+    /**
+     * The number of lines to show above the error. default: 2
+     */
+    linesAbove?: number,
+
+    /**
+     * The number of lines to show below the error. default: 3
+     */
+    linesBelow?: number,
+
+    /**
+     * Forcibly syntax highlight the code as JavaScript (for non-terminals);
+     * overrides highlightCode.
+     * default: false
+     */
+    forceColor?: boolean,
+
+    /**
+     * Pass in a string to be displayed inline (if possible) next to the
+     * highlighted location in the code. If it can't be positioned inline,
+     * it will be placed above the code frame.
+     * default: nothing
+     */
+    message?: string,
+    ...
+  }
+
+  /**
+   * Generate errors that contain a code frame that point to source locations.
+   * @param rawLines Raw lines to frame
+   * @param lineNumber Line number (1 indexed)
+   * @param colNumber Column number
+   * @param options Additional options
+   * @returns Framed code
+   */
+  declare export default function codeFrame(
+    rawLines: string,
+    lineNumber: number,
+    colNumber: number,
+    options?: BabelCodeFrameOptions
+  ): string;
+}

cli/src/commands/__tests__/__install-fixtures__/installNpmLibDef/fakeCacheRepo/experimental/definitions/@babel/code-frame/code-frame_v7.x.x/flow_v0.84.x-/tests/test_code-frame_v7.x.x.js

@@ -0,0 +1,32 @@
+// @flow
+
+import { describe, it } from 'flow-typed-test';
+import codeFrame, { codeFrameColumns } from '@babel/code-frame';
+
+const code = `
+  const number = 1;
+  var string = 'foo';
+  function print(name: string) {
+    console.log(string + name);
+  }
+`;
+
+it('codeFrame', () => {
+  codeFrame(code, 5, 22);
+  codeFrame(code, 5, 22, { forceColor: true });
+  codeFrame(code, 2, 2, { highlightCode: true });
+});
+
+it('codeFrameColumns', () => {
+  codeFrameColumns(code, { start: { line: 5, column: 22 } });
+  codeFrameColumns(
+    code,
+    { start: { line: 5, column: 22 } },
+    { forceColor: true }
+  );
+  codeFrameColumns(
+    code,
+    { start: { line: 2, column: 2 } },
+    { highlightCode: true }
+  );
+});

cli/src/commands/__tests__/install-test.js

@@ -234,13 +234,13 @@ describe('install (command)', () => {
           }),
         ]);
         await gitInit(FAKE_CACHE_REPO_DIR),
-          await gitAdd(FAKE_CACHE_REPO_DIR, 'definitions');
+          await gitAdd(FAKE_CACHE_REPO_DIR, 'experimental');
         await gitCommit(FAKE_CACHE_REPO_DIR, 'FIRST');
 
         setCustomCacheDir(FAKE_CACHE_DIR);
 
         const availableLibDefs = await getNpmLibDefs(
-          path.join(FAKE_CACHE_REPO_DIR, 'definitions'),
+          path.join(FAKE_CACHE_REPO_DIR, 'experimental'),
         );
 
         await installNpmLibDef(availableLibDefs[0], FLOWTYPED_DIR, false);
@@ -283,7 +283,7 @@ describe('install (command)', () => {
             gitConfig(FAKE_CACHE_REPO_DIR, 'user.name', 'Test Author'),
             gitConfig(FAKE_CACHE_REPO_DIR, 'user.email', 'test@flow-typed.org'),
           ]);
-        await gitAdd(FAKE_CACHE_REPO_DIR, 'definitions');
+        await gitAdd(FAKE_CACHE_REPO_DIR, 'experimental');
         await gitCommit(FAKE_CACHE_REPO_DIR, 'FIRST');
 
         setCustomCacheDir(FAKE_CACHE_DIR);

cli/src/commands/runTests.js

@@ -53,7 +53,7 @@ type TestGroup = {
  * structs. Each TestGroup represents a Package/PackageVersion/FlowVersion
  * directory.
  */
-const basePathRegex = new RegExp('definitions/npm/(@[^/]*/)?[^/]*/?');
+const basePathRegex = new RegExp('experimental/definitions/(@[^/]*/)?[^/]*/?');
 async function getTestGroups(
   repoDirPath,
   onlyChanged: boolean = false,
@@ -75,9 +75,7 @@ async function getTestGroups(
     libDefs = libDefs.filter(def => changedDefs.includes(def.pkgName));
   }
   return libDefs.map(libDef => {
-    const groupID = `${libDef.pkgName}_${libDef.pkgVersionStr}/${
-      libDef.flowVersionStr
-    }`;
+    const groupID = `${libDef.pkgName}_${libDef.pkgVersionStr}/${libDef.flowVersionStr}`;
     return {
       id: groupID,
       testFilePaths: libDef.testFilePaths,
@@ -293,10 +291,15 @@ async function getCachedFlowBinVersions(
 }
 
 async function writeFlowConfig(repoDirPath, testDirPath, libDefPath, version) {
+  const basePath = path.dirname(libDefPath);
   const destFlowConfigPath = path.join(testDirPath, '.flowconfig');
+  const pkg = path.join(basePath, 'package.json');
+  const deps = Object.keys(require(pkg).dependencies)
+    .map(key => path.join(basePath, 'node_modules', key, 'index.js'));
 
   const flowConfigData = [
     '[libs]',
+    ...deps,
     path.basename(libDefPath),
     path.join(repoDirPath, '..', '__util__', 'tdd_framework.js'),
     '',
@@ -540,9 +543,7 @@ async function runTestGroup(
     );
 
     if (lowestCapableFlowVersion !== lowestFlowVersionRan) {
-      console.log(`Tests for ${
-        testGroup.id
-      } ran successfully on flow ${lowestCapableFlowVersion}.
+      console.log(`Tests for ${testGroup.id} ran successfully on flow ${lowestCapableFlowVersion}.
         Consider setting ${lowestCapableFlowVersion} as the lower bound!`);
     }
 
@@ -663,10 +664,10 @@ export async function run(argv: Args): Promise<number> {
 
   const cwd = process.cwd();
   const basePath = argv.path ? String(argv.path) : cwd;
-  const cwdDefsNPMPath = path.join(basePath, 'definitions', 'npm');
+  const cwdDefsNPMPath = path.join(basePath, 'experimental', 'definitions');
   let repoDirPath = (await fs.exists(cwdDefsNPMPath))
     ? cwdDefsNPMPath
-    : path.join(__dirname, '..', '..', '..', 'definitions', 'npm');
+    : path.join(__dirname, '..', '..', '..', 'experimental', 'definitions');
 
   if (onlyChanged) {
     console.log(

cli/src/lib/cacheRepoUtils.js

@@ -127,7 +127,7 @@ export function getCacheRepoDir() {
 export async function verifyCLIVersion(): Promise<void> {
   const metadataPath = path.join(
     getCacheRepoDir(),
-    'definitions',
+    'experimental',
     '.cli-metadata.json',
   );
   const metadata = JSON.parse(String(await fs.readFile(metadataPath)));

cli/src/lib/libDefs.js

@@ -152,9 +152,15 @@ export {
 };
 
 async function addLibDefs(pkgDirPath, libDefs: Array<LibDef>) {
-  const parsedDirItem = parseRepoDirItem(pkgDirPath);
-  (await parseLibDefsFromPkgDir(parsedDirItem, pkgDirPath)).forEach(libDef =>
-    libDefs.push(libDef),
+  const versionedRepoDirs = await fs.readdir(pkgDirPath);
+  const parsedDirItems = versionedRepoDirs
+    .map(dir => path.join(pkgDirPath, dir))
+    .map(parseRepoDirItem);
+  await Promise.all(
+    parsedDirItems.map(async parsedDirItem => {
+      const libdefs = await parseLibDefsFromPkgDir(parsedDirItem);
+      libdefs.forEach(libDef => libDefs.push(libDef));
+    }),
   );
 }
 
@@ -209,10 +215,11 @@ function parsePkgFlowDirVersion(pkgFlowDirPath): FlowVersion {
  * on disk, scan the directory and generate a list of LibDefs for each
  * flow-versioned definition file.
  */
-async function parseLibDefsFromPkgDir(
-  {pkgName, pkgVersion},
-  pkgDirPath,
-): Promise<Array<LibDef>> {
+async function parseLibDefsFromPkgDir({
+  pkgName,
+  pkgVersion,
+  path: pkgDirPath,
+}): Promise<Array<LibDef>> {
   const pkgVersionStr = versionToString(pkgVersion);
   const pkgDirItems = await fs.readdir(pkgDirPath);
 
@@ -251,9 +258,7 @@ async function parseLibDefsFromPkgDir(
   await P.all(
     flowDirs.map(async ([flowDirPath, flowVersion]) => {
       const testFilePaths = [].concat(commonTestFiles);
-      const basePkgName =
-        pkgName.charAt(0) === '@' ? pkgName.split(path.sep).pop() : pkgName;
-      const libDefFileName = `${basePkgName}_${pkgVersionStr}.js`;
+      const libDefFileName = `index.js`;
       let libDefFilePath;
       (await fs.readdir(flowDirPath)).forEach(flowDirItem => {
         const flowDirItemPath = path.join(flowDirPath, flowDirItem);
@@ -280,11 +285,25 @@ async function parseLibDefsFromPkgDir(
             testFilePaths.push(flowDirItemPath);
           }
         } else {
+          const isValidTestDir =
+            path.basename(flowDirItemPath) === 'tests' ||
+            path.basename(flowDirItemPath) === 'node_modules';
+          if (isValidTestDir) return;
           const error = 'Unexpected directory item: ' + flowDirItemPath;
           throw new ValidationError(error);
         }
       });
 
+      (await fs.readdir(path.join(flowDirPath, 'tests'))).forEach(
+        testFileItem => {
+          const flowDirItemPath = path.join(flowDirPath, 'tests', testFileItem);
+          const isValidTestFile = validateTestFile(flowDirItemPath);
+          if (isValidTestFile) {
+            testFilePaths.push(flowDirItemPath);
+          }
+        },
+      );
+
       if (libDefFilePath == null) {
         libDefFilePath = path.join(flowDirPath, libDefFileName);
         if (pkgName !== 'ERROR') {
@@ -326,7 +345,7 @@ export function parseRepoDirItem(dirItemPath: string) {
   const item = path
     .dirname(dirItemPath)
     .split(path.sep)
-    .pop();
+    .splice(-2, 2)[0];
   if (item.charAt(0) === '@') {
     pkgName = `${item}${path.sep}${pkgName}`;
   }
@@ -338,7 +357,11 @@ export function parseRepoDirItem(dirItemPath: string) {
     prerel = prerel.substr(1);
   }
 
-  return {pkgName, pkgVersion: {major, minor, patch, prerel}};
+  return {
+    pkgName,
+    pkgVersion: {major, minor, patch, prerel},
+    path: dirItemPath,
+  };
 }
 
 /**
@@ -415,12 +438,16 @@ function writeVerbose(stream, msg, writeNewline = true) {
  * If the repo checkout does not exist or is out of date, it will be
  * created/updated automatically first.
  */
-const CACHE_REPO_DEFS_DIR = path.join(CACHE_REPO_DIR, 'definitions', 'npm');
+const CACHE_REPO_DEFS_DIR = path.join(
+  CACHE_REPO_DIR,
+  'experimental',
+  'definitions',
+);
 export async function getCacheLibDefs(
   verbose?: VerboseOutput = process.stdout,
 ) {
   await ensureCacheRepo(verbose);
-  await verifyCLIVersion(path.join(CACHE_REPO_DIR, 'definitions'));
+  await verifyCLIVersion(path.join(CACHE_REPO_DIR, 'experimental'));
   return getLibDefs(CACHE_REPO_DEFS_DIR);
 }