gulpfile.ts

/**
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See License.txt in the project root for
 * license information.
 */

import { contains, getArgument, gitDiff, GitDiffResult, gitStatus, GitStatusResult, joinPath, normalize, npmInstall, npmRun, NPMScope, NPMViewResult, RunOptions, StringMap } from "@ts-common/azure-js-dev-tools";
import * as fs from "fs";
import gulp from "gulp";
import * as path from "path";
import PluginError from "plugin-error";
import { Argv, CommandLineOptions, getCommandLineOptions } from "./.scripts/commandLine";
import { endsWith, getPackageFolderPaths, packagesToIgnore } from "./.scripts/common";
import { generateSdk, setAutoPublish, setVersion } from "./.scripts/gulp";
import { Logger } from "./.scripts/logger";
import { findMissingSdks } from "./.scripts/packages";
import { getPackageFolderPathFromPackageArgument } from "./.scripts/readme";

enum PackagesToPack {
  All,
  DifferentVersion,
  BranchHasChanges
}

function getPackagesToPackArgument(toPackArgument: string | undefined): PackagesToPack {
  let result: PackagesToPack = PackagesToPack.BranchHasChanges;
  if (toPackArgument) {
    const toPackArgumentLower: string = toPackArgument.toLowerCase();
    for (const option in PackagesToPack) {
      if (option.toLowerCase() === toPackArgumentLower) {
        result = PackagesToPack[option] as any;
        break;
      }
    }
  }
  return result;
}

const args: CommandLineOptions = getCommandLineOptions();
const _logger: Logger = Logger.get();

const azureSDKForJSRepoRoot: string = getArgument("azure-sdk-for-js-repo-root", { defaultValue: __dirname })!;
const rawToPack: string | undefined = getArgument("to-pack");
let toPack: PackagesToPack = getPackagesToPackArgument(rawToPack);
const headReference: string | undefined = getArgument("head-reference", { environmentVariableName: "headReference" });
const baseReference: string | undefined = getArgument("base-reference", { environmentVariableName: "baseReference" });

function getDropFolderPath(): string {
  let result: string | undefined = getArgument("drop");
  if (!result) {
    result = "drop";
  }
  if (!path.isAbsolute(result)) {
    result = path.join(azureSDKForJSRepoRoot, result);
  }
  return result;
}

const dropFolderPath: string = getDropFolderPath();
if (!fs.existsSync(dropFolderPath)) {
  fs.mkdirSync(dropFolderPath);
}

gulp.task('default', async () => {
  _logger.log('gulp build --package <package-name>');
  _logger.log('  --package');
  _logger.log('    NPM package to run "npm run build" on.');
  _logger.log();
  _logger.log('gulp install --package <package name>');
  _logger.log('  --package');
  _logger.log('    NPM package to run "npm install" on.');
  _logger.log();
  _logger.log('gulp codegen [--azure-rest-api-specs-root <azure-rest-api-specs root>] [--use <autorest.typescript root>] [--package <package name>]');
  _logger.log('  --azure-rest-api-specs-root');
  _logger.log('    Root location of the local clone of the azure-rest-api-specs-root repository.');
  _logger.log('  --use');
  _logger.log('    Root location of autorest.typescript repository. If this is not specified, then the latest installed generator for TypeScript will be used.');
  _logger.log('  --package');
  _logger.log('    NPM package to regenerate. If no package is specified, then all packages will be regenerated.');
  _logger.log();
  _logger.log('gulp pack [--package <package name>] [--whatif] [--to-pack <to-pack option>] [--drop <drop folder path>]');
  _logger.log('  --package');
  _logger.log('    The name of the package to pack. If no package is specified, then all packages will be packed.');
  _logger.log('  --whatif');
  _logger.log('    Don\'t actually pack packages, but just indicate which packages would be packed.');
  _logger.log("  --to-pack");
  _logger.log(`    Which packages should be packed. Options are "All", "DifferentVersion", "BranchHasChanges".`);
  _logger.log(`  --drop`);
  _logger.log(`    The folder where packed tarballs will be put. Defaults to "<azure-sdk-for-js-root>/drop/".`);
});

gulp.task("install", async () => {
  _logger.log(`Passed arguments: ${Argv.print()}`);
  const argv: (Argv.PackageOptions & Argv.RepositoryOptions)
    = Argv.construct(Argv.Options.Package, Argv.Options.Repository)
      .usage("Example: gulp install --package @azure/arm-mariadb")
      .argv as any;

  const packageFolderPath: string | undefined = await getPackageFolderPathFromPackageArgument(
    argv.package,
    argv.azureRestAPISpecsRoot,
    argv.azureSDKForJSRepoRoot,
  );
  if (packageFolderPath) {
    npmInstall({ executionFolderPath: packageFolderPath });
  }
});

gulp.task("build", async () => {
  _logger.log(`Passed arguments: ${Argv.print()}`);
  const argv: (Argv.PackageOptions & Argv.RepositoryOptions)
    = Argv.construct(Argv.Options.Package, Argv.Options.Repository)
      .usage("Example: gulp build --package @azure/arm-mariadb")
      .argv as any;

  const packageFolderPath: string | undefined = await getPackageFolderPathFromPackageArgument(
    argv.package,
    argv.azureRestAPISpecsRoot,
    argv.azureSDKForJSRepoRoot,
  );
  if (packageFolderPath) {
    npmRun("build", { executionFolderPath: packageFolderPath });
  }
});

// This task is used to generate libraries based on the mappings specified above.
gulp.task('codegen', async () => {
  interface CodegenOptions {
    debugger: boolean | undefined;
    use: string | undefined;
  };

  _logger.log(`Passed arguments: ${Argv.print()}`);
  const argv: (CodegenOptions & Argv.PackageOptions & Argv.RepositoryOptions)
    = Argv.construct(Argv.Options.Package, Argv.Options.Repository)
      .options({
        "debugger": {
          boolean: true,
          alias: ["d", "use-debugger"],
          description: "Enables debugger attaching to autorest.typescript process"
        },
        "use": {
          string: true,
          description: "Specifies location for the generator to use"
        }
      })
      .usage("Example: gulp codegen --package @azure/arm-mariadb")
      .argv as any;

  await generateSdk(argv.azureRestAPISpecsRoot, argv.azureSDKForJSRepoRoot, argv.package, argv.use, argv.debugger);
});

function pack(): void {
  const runOptions: RunOptions = {
    log: (text: string) => _logger.logTrace(text),
    showCommand: true,
    showOutput: true
  };

  let errorPackages = 0;
  let upToDatePackages = 0;
  let packedPackages = 0;
  let skippedPackages = 0;

  const changedFiles: string[] = [];

  if (toPack === PackagesToPack.BranchHasChanges) {
    let packBaseReference: string | undefined = baseReference;
    if (!packBaseReference) {
      packBaseReference = "master";
      _logger.log(`No base-reference argument specified on command line or in environment variables. Defaulting to "${packBaseReference}".`);
    }

    let packHeadReference: string | undefined = headReference;
    if (!packHeadReference) {
      const statusResult: GitStatusResult = gitStatus(runOptions);
      packHeadReference = statusResult.localBranch!;
      _logger.log(`No head-reference argument specified on command line or in environment variables. Defaulting to "${packHeadReference}".`);

      const modifiedFiles: string[] | undefined = statusResult.modifiedFiles;
      if (modifiedFiles) {
        changedFiles.push(...modifiedFiles);
      }
    }

    if (packBaseReference === packHeadReference) {
      if (rawToPack) {
        _logger.logWarn(`The base-reference "${packBaseReference}" is equal to the head-reference "${packHeadReference}". This will result in nothing getting packed because there won't be any changes detected. Please change either the base or head-reference.`);
      } else {
        toPack = PackagesToPack.DifferentVersion;
        _logger.log(`The base-reference "${packBaseReference}" is equal to the head-reference "${packHeadReference}" which means there won't be any changes to pack. Switching "to-pack" to be "${PackagesToPack[toPack]}".`);
      }
    } else {
      const diffResult: GitDiffResult = gitDiff(packBaseReference, packHeadReference, runOptions);
      changedFiles.push(...diffResult.filesChanged);
      if (!changedFiles || changedFiles.length === 0) {
        _logger.logTrace(`Found no changes between "${packBaseReference}" and "${packHeadReference}".`);
      } else {
        _logger.logTrace(`Found the following changed files`)
        for (const changedFilePath of changedFiles) {
          _logger.logTrace(changedFilePath);
        }
      }
    }
  }

  const packageFolderRoot: string = path.resolve(__dirname, "sdk");
  _logger.logTrace(`INFO: Searching for package folders in ${packageFolderRoot}`);
  const packageFolderPaths: string[] | undefined = getPackageFolderPaths(packageFolderRoot);
  if (!packageFolderPaths) {
    _logger.logTrace(`INFO: The folder ${packageFolderPaths} doesn't exist.`);
  } else {
    for (const packageFolderPath of packageFolderPaths) {
      _logger.logTrace(`INFO: Processing ${packageFolderPath}`);

      const npm = new NPMScope({ executionFolderPath: packageFolderPath });
      const packageJsonFilePath: string = joinPath(packageFolderPath, "package.json");
      const packageJson: { [propertyName: string]: any } = require(packageJsonFilePath);
      const packageName: string = packageJson.name;

      if (packagesToIgnore.indexOf(packageName) !== -1) {
        _logger.log(`INFO: Skipping package ${packageName}`);
        ++skippedPackages;
      } else if (!args.package || args.package === packageName || endsWith(packageName, `-${args.package}`)) {
        const localPackageVersion: string = packageJson.version;
        if (!localPackageVersion) {
          _logger.log(`ERROR: "${packageJsonFilePath}" doesn't have a version specified.`);
          errorPackages++;
        }
        else {
          let shouldPack: boolean = false;

          if (toPack === PackagesToPack.All) {
            shouldPack = true;
          } else if (toPack === PackagesToPack.DifferentVersion) {
            let npmPackageVersion: string | undefined;
            try {
              const npmViewResult: NPMViewResult = npm.view({ packageName, ...runOptions, showCommand: false, showOutput: false });
              const distTags: StringMap<string> | undefined = npmViewResult["dist-tags"];
              npmPackageVersion = distTags && distTags["latest"];
            }
            catch (error) {
              // This happens if the package doesn't exist in NPM.
            }

            _logger.logTrace(`Local version: ${localPackageVersion}, NPM version: ${npmPackageVersion}`);
            shouldPack = localPackageVersion !== npmPackageVersion;
          } else if (toPack === PackagesToPack.BranchHasChanges) {
            const packageFolderPathWithSep: string = normalize(packageFolderPath + path.posix.sep);
            shouldPack = !!changedFiles && contains(changedFiles, (changedFilePath: string) => normalize(changedFilePath).startsWith(packageFolderPathWithSep));
          }

          if (!shouldPack) {
            upToDatePackages++;
          } else {
            _logger.log(`Packing package "${packageName}" with version "${localPackageVersion}"...${args.whatif ? " (SKIPPED)" : ""}`);
            if (!args.whatif) {
              try {
                npm.pack(runOptions);
                const packFileName = `${packageName.replace("/", "-").replace("@", "")}-${localPackageVersion}.tgz`
                const packFilePath = path.join(packageFolderPath, packFileName);
                fs.renameSync(packFilePath, path.join(dropFolderPath, packFileName));
                _logger.log(`Filename: ${packFileName}`);
                packedPackages++;
              }
              catch (error) {
                errorPackages++;
              }
            } else {
              skippedPackages++;
            }
          }
        }
      }
    }
  }


  function padLeft(value: number, minimumWidth: number, padCharacter: string = " "): string {
    let result: string = value.toString();
    while (result.length < minimumWidth) {
      result = padCharacter + result;
    }
    return result;
  }
  const minimumWidth: number = Math.max(errorPackages, upToDatePackages, packedPackages, skippedPackages).toString().length;
  _logger.log();
  _logger.log(`Error packages:      ${padLeft(errorPackages, minimumWidth)}`);
  _logger.log(`Up to date packages: ${padLeft(upToDatePackages, minimumWidth)}`);
  _logger.log(`Packed packages:     ${padLeft(packedPackages, minimumWidth)}`);
  _logger.log(`Skipped packages:    ${padLeft(skippedPackages, minimumWidth)}`);

  if (errorPackages !== 0) {
    throw new PluginError("pack", { message: "Some packages failed to pack." });
  }
}

gulp.task('pack', async () => pack());

gulp.task("find-missing-sdks", async () => {
  try {
    _logger.log(`Passed arguments: ${Argv.print()}`);
    const argv: Argv.RepositoryOptions
      = Argv.construct(Argv.Options.Repository)
        .usage("Example: gulp find-missing-sdks")
        .argv as any;

    const azureRestApiSpecsRepositoryPath = argv.azureRestAPISpecsRoot;
    _logger.log(`Found azure-rest-api-specs repository in ${azureRestApiSpecsRepositoryPath}`);

    await findMissingSdks(azureRestApiSpecsRepositoryPath);
  } catch (error) {
    _logger.logError(error);
  }
});

gulp.task("set-autopublish", async () => {
  _logger.log(`Passed arguments: ${Argv.print()}`);
  const argv: Argv.RepositoryOptions & Argv.FilterOptions
    = Argv.construct(Argv.Options.Repository, Argv.Options.Filter)
      .usage("Example: gulp set-autopublish")
      .argv as any;

  await setAutoPublish(argv.azureSDKForJSRepoRoot, argv.include, argv.exclude || /@azure\/(keyvault|template|service-bus)/);
});

gulp.task("set-version", async () => {
  _logger.log(`Passed arguments: ${Argv.print()}`);
  const argv: Argv.RepositoryOptions & Argv.FilterOptions
    = Argv.construct(Argv.Options.Repository, Argv.Options.Filter)
      .usage("Example: gulp set-version")
      .argv as any;

  await setVersion(argv.azureSDKForJSRepoRoot, argv.include, argv.exclude || /@azure\/(keyvault|template|service-bus)/);
});