Add initial support for new env handling (#10525)

* Add initial support for new env config file

* Fix serverless processEnv call when no env is provided

* Add missing await for test method

* Update env config to .env.json and add dotenv loading

* ncc dotenv package

* Update type

* Update with new discussed behavior removing .env.json

* Update hot-reloader createEntrypoints

* Make sure .env is loaded before next.config.js

* Add tests for all separate .env files

* Remove comments

* Add override tests

* Add test for overriding env vars based on local environment

* Add support for .env.test

* Apply suggestions from code review

Co-Authored-By: Joe Haddad <joe.haddad@zeit.co>

* Use chalk for env loaded message

* Remove constant as it’s not needed

* Update test

* Update errsh, taskr, and CNA template ignores

* Make sure to only consider undefined missing

* Remove old .env ignore

* Update to not populate process.env with loaded env

* Add experimental flag and add loading of global env values

Co-authored-by: Tim Neutkens <timneutkens@me.com>
Co-authored-by: Joe Haddad <joe.haddad@zeit.co>

errors/missing-env-value.md

@@ -0,0 +1,28 @@
+# Missing Env Value
+
+#### Why This Error Occurred
+
+One of your pages' config requested an env value that wasn't populated.
+
+```js
+// pages/index.js
+export const config = {
+  // this value isn't provided in `.env`
+  env: ['MISSING_KEY'],
+}
+```
+
+```
+// .env (notice no `MISSING_KEY` provided here)
+NOTION_KEY='...'
+```
+
+#### Possible Ways to Fix It
+
+Either remove the requested env value from the page's config, populate it in your `.env` file, or manually populate it in your environment before running `next dev` or `next build`.
+
+### Useful Links
+
+- [dotenv](https://npmjs.com/package/dotenv)
+- [dotenv-expand](https://npmjs.com/package/dotenv-expand)
+- [Environment Variables](https://en.wikipedia.org/wiki/Environment_variable)

packages/create-next-app/templates/default/gitignore

@@ -17,9 +17,14 @@
 
 # misc
 .DS_Store
-.env*
 
 # debug
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# local env files
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local

packages/next/build/index.ts

@@ -71,6 +71,7 @@ import {
 } from './utils'
 import getBaseWebpackConfig from './webpack-config'
 import { writeBuildId } from './write-build-id'
+import { loadEnvConfig } from '../lib/load-env-config'
 
 const fsAccess = promisify(fs.access)
 const fsUnlink = promisify(fs.unlink)
@@ -110,6 +111,9 @@ export default async function build(dir: string, conf = null): Promise<void> {
     )
   }
 
+  // attempt to load global env values so they are available in next.config.js
+  loadEnvConfig(dir)
+
   const config = loadConfig(PHASE_PRODUCTION_BUILD, dir, conf)
   const { target } = config
   const buildId = await generateBuildId(config.generateBuildId, nanoid)

packages/next/build/webpack-config.ts

@@ -707,6 +707,17 @@ export default async function getBaseWebpackConfig(
       // This plugin makes sure `output.filename` is used for entry chunks
       new ChunkNamesPlugin(),
       new webpack.DefinePlugin({
+        ...(config.experimental.pageEnv
+          ? Object.keys(process.env).reduce(
+              (prev: { [key: string]: string }, key: string) => {
+                if (key.startsWith('NEXT_APP_')) {
+                  prev[key] = process.env[key]!
+                }
+                return prev
+              },
+              {}
+            )
+          : {}),
         ...Object.keys(config.env).reduce((acc, key) => {
           if (/^(?:NODE_.+)|^(?:__.+)$/i.test(key)) {
             throw new Error(

packages/next/build/webpack/loaders/next-serverless-loader.ts

@@ -181,6 +181,7 @@ const nextServerlessLoader: loader.Loader = function() {
             Object.assign({}, parsedUrl.query, params ),
             resolver,
             ${encodedPreviewProps},
+            process.env,
             onError
           )
         } catch (err) {
@@ -257,6 +258,7 @@ const nextServerlessLoader: loader.Loader = function() {
         assetPrefix: "${assetPrefix}",
         runtimeConfig: runtimeConfig.publicRuntimeConfig || {},
         previewProps: ${encodedPreviewProps},
+        env: process.env,
         ..._renderOpts
       }
       let _nextData = false

packages/next/export/index.ts

@@ -35,6 +35,7 @@ import loadConfig, {
 import { eventCliSession } from '../telemetry/events'
 import { Telemetry } from '../telemetry/storage'
 import { normalizePagePath } from '../next-server/server/normalize-page-path'
+import { loadEnvConfig } from '../lib/load-env-config'
 
 const copyFile = promisify(copyFileOrig)
 const mkdir = promisify(mkdirOrig)
@@ -230,6 +231,7 @@ export default async function(
     dir,
     buildId,
     nextExport: true,
+    env: loadEnvConfig(dir),
     assetPrefix: nextConfig.assetPrefix.replace(/\/$/, ''),
     distDir,
     dev: false,

packages/next/lib/find-pages-dir.ts

@@ -1,7 +1,7 @@
 import fs from 'fs'
 import path from 'path'
 
-const existsSync = (f: string): boolean => {
+export const existsSync = (f: string): boolean => {
   try {
     fs.accessSync(f, fs.constants.F_OK)
     return true

packages/next/lib/load-env-config.ts

@@ -0,0 +1,84 @@
+import fs from 'fs'
+import path from 'path'
+import chalk from 'chalk'
+import dotenvExpand from 'next/dist/compiled/dotenv-expand'
+import dotenv, { DotenvConfigOutput } from 'next/dist/compiled/dotenv'
+import findUp from 'find-up'
+
+export type Env = { [key: string]: string }
+
+export function loadEnvConfig(dir: string, dev?: boolean): Env | false {
+  const packageJson = findUp.sync('package.json', { cwd: dir })
+
+  // only do new env loading if dotenv isn't installed since we
+  // can't check for an experimental flag in next.config.js
+  // since we want to load the env before loading next.config.js
+  if (packageJson) {
+    const { dependencies, devDependencies } = require(packageJson)
+    const allPackages = Object.keys({
+      ...dependencies,
+      ...devDependencies,
+    })
+
+    if (allPackages.some(pkg => pkg === 'dotenv')) {
+      return false
+    }
+  } else {
+    // we should always have a package.json but disable in case we don't
+    return false
+  }
+
+  const isTest = process.env.NODE_ENV === 'test'
+  const mode = isTest ? 'test' : dev ? 'development' : 'production'
+  const dotenvFiles = [
+    `.env.${mode}.local`,
+    `.env.${mode}`,
+    // Don't include `.env.local` for `test` environment
+    // since normally you expect tests to produce the same
+    // results for everyone
+    mode !== 'test' && `.env.local`,
+    '.env',
+  ].filter(Boolean) as string[]
+
+  const combinedEnv: Env = {
+    ...(process.env as any),
+  }
+
+  for (const envFile of dotenvFiles) {
+    // only load .env if the user provided has an env config file
+    const dotEnvPath = path.join(dir, envFile)
+
+    try {
+      const contents = fs.readFileSync(dotEnvPath, 'utf8')
+      let result: DotenvConfigOutput = {}
+      result.parsed = dotenv.parse(contents)
+
+      result = dotenvExpand(result)
+
+      if (result.parsed) {
+        console.log(`> ${chalk.cyan.bold('Info:')} Loaded env from ${envFile}`)
+      }
+
+      Object.assign(combinedEnv, result.parsed)
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        console.log(
+          `> ${chalk.cyan.bold('Error: ')} Failed to load env from ${envFile}`,
+          err
+        )
+      }
+    }
+  }
+
+  // load global env values prefixed with `NEXT_APP_` to process.env
+  for (const key of Object.keys(combinedEnv)) {
+    if (
+      key.startsWith('NEXT_APP_') &&
+      typeof process.env[key] === 'undefined'
+    ) {
+      process.env[key] = combinedEnv[key]
+    }
+  }
+
+  return combinedEnv
+}

packages/next/next-server/lib/utils.ts

@@ -4,6 +4,7 @@ import { ComponentType } from 'react'
 import { format, URLFormatOptions, UrlObject } from 'url'
 import { ManifestItem } from '../server/load-components'
 import { NextRouter } from './router/router'
+import { Env } from '../../lib/load-env-config'
 
 /**
  * Types used by both next and next-server
@@ -186,6 +187,8 @@ export type NextApiRequest = IncomingMessage & {
   }
 
   body: any
+
+  env: Env
 }
 
 /**

packages/next/next-server/server/api-utils.ts

@@ -8,6 +8,8 @@ import { isResSent, NextApiRequest, NextApiResponse } from '../lib/utils'
 import { decryptWithSecret, encryptWithSecret } from './crypto-utils'
 import { interopDefault } from './load-components'
 import { Params } from './router'
+import { collectEnv } from './utils'
+import { Env } from '../../lib/load-env-config'
 
 export type NextApiRequestCookies = { [key: string]: string }
 export type NextApiRequestQuery = { [key: string]: string | string[] }
@@ -24,26 +26,23 @@ export async function apiResolver(
   params: any,
   resolverModule: any,
   apiContext: __ApiPreviewProps,
+  env: Env | false,
   onError?: ({ err }: { err: any }) => Promise<void>
 ) {
   const apiReq = req as NextApiRequest
   const apiRes = res as NextApiResponse
 
   try {
-    let config: PageConfig = {}
-    let bodyParser = true
     if (!resolverModule) {
       res.statusCode = 404
       res.end('Not Found')
       return
     }
+    const config: PageConfig = resolverModule.config || {}
+    const bodyParser = config.api?.bodyParser !== false
+
+    apiReq.env = env ? collectEnv(req.url!, env, config.env) : {}
 
-    if (resolverModule.config) {
-      config = resolverModule.config
-      if (config.api && config.api.bodyParser === false) {
-        bodyParser = false
-      }
-    }
     // Parsing of cookies
     setLazyProp({ req: apiReq }, 'cookies', getCookieParser(req))
     // Parsing query string

packages/next/next-server/server/config.ts

@@ -54,6 +54,7 @@ const defaultConfig: { [key: string]: any } = {
     workerThreads: false,
     basePath: '',
     sassOptions: {},
+    pageEnv: false,
   },
   future: {
     excludeDefaultMomentLocales: false,

packages/next/next-server/server/next-server.ts

@@ -61,6 +61,7 @@ import {
   setSprCache,
 } from './spr-cache'
 import { isBlockedPage } from './utils'
+import { loadEnvConfig, Env } from '../../lib/load-env-config'
 
 const getCustomRouteMatcher = pathMatch(true)
 
@@ -117,6 +118,7 @@ export default class Server {
     documentMiddlewareEnabled: boolean
     hasCssMode: boolean
     dev?: boolean
+    env: Env | false
     previewProps: __ApiPreviewProps
     customServer?: boolean
     ampOptimizerConfig?: { [key: string]: any }
@@ -145,6 +147,8 @@ export default class Server {
     this.dir = resolve(dir)
     this.quiet = quiet
     const phase = this.currentPhase()
+    const env = loadEnvConfig(this.dir, dev)
+
     this.nextConfig = loadConfig(phase, this.dir, conf)
     this.distDir = join(this.dir, this.nextConfig.distDir)
     this.publicDir = join(this.dir, CLIENT_PUBLIC_FILES_PATH)
@@ -171,6 +175,7 @@ export default class Server {
       staticMarkup,
       buildId: this.buildId,
       generateEtags,
+      env: this.nextConfig.experimental.pageEnv && env,
       previewProps: this.getPreviewProps(),
       customServer: customServer === true ? true : undefined,
       ampOptimizerConfig: this.nextConfig.experimental.amp?.optimizer,
@@ -684,6 +689,7 @@ export default class Server {
       query,
       pageModule,
       this.renderOpts.previewProps,
+      this.renderOpts.env,
       this.onErrorMiddleware
     )
     return true

packages/next/next-server/server/render.tsx

@@ -38,6 +38,8 @@ import { tryGetPreviewData, __ApiPreviewProps } from './api-utils'
 import { getPageFiles } from './get-page-files'
 import { LoadComponentsReturnType, ManifestItem } from './load-components'
 import optimizeAmp from './optimize-amp'
+import { collectEnv } from './utils'
+import { Env } from '../../lib/load-env-config'
 import { UnwrapPromise } from '../../lib/coalesced-function'
 import { GetStaticProps, GetServerSideProps } from '../../types'
 
@@ -154,6 +156,7 @@ export type RenderOptsPartial = {
   isDataReq?: boolean
   params?: ParsedUrlQuery
   previewProps: __ApiPreviewProps
+  env: Env | false
 }
 
 export type RenderOpts = LoadComponentsReturnType & RenderOptsPartial
@@ -288,6 +291,7 @@ export async function renderToHTML(
     staticMarkup = false,
     ampPath = '',
     App,
+    env = {},
     Document,
     pageConfig = {},
     DocumentMiddleware,
@@ -303,6 +307,8 @@ export async function renderToHTML(
     previewProps,
   } = renderOpts
 
+  const curEnv = env ? collectEnv(pathname, env, pageConfig.env) : {}
+
   const callMiddleware = async (method: string, args: any[], props = false) => {
     let results: any = props ? {} : []
 
@@ -503,6 +509,7 @@ export async function renderToHTML(
 
       try {
         data = await getStaticProps!({
+          env: curEnv,
           ...(pageIsDynamic ? { params: query as ParsedUrlQuery } : undefined),
           ...(previewData !== false
             ? { preview: true, previewData: previewData }
@@ -585,6 +592,7 @@ export async function renderToHTML(
           req,
           res,
           query,
+          env: curEnv,
           ...(pageIsDynamic ? { params: params as ParsedUrlQuery } : undefined),
           ...(previewData !== false
             ? { preview: true, previewData: previewData }

packages/next/next-server/server/utils.ts

@@ -1,4 +1,5 @@
 import { BLOCKED_PAGES } from '../lib/constants'
+import { Env } from '../../lib/load-env-config'
 
 export function isBlockedPage(pathname: string): boolean {
   return BLOCKED_PAGES.indexOf(pathname) !== -1
@@ -14,3 +15,28 @@ export function cleanAmpPath(pathname: string): string {
   pathname = pathname.replace(/\?$/, '')
   return pathname
 }
+
+export function collectEnv(page: string, env: Env, pageEnv?: string[]): Env {
+  const missingEnvKeys = new Set()
+  const collected = pageEnv
+    ? pageEnv.reduce((prev: Env, key): Env => {
+        if (typeof env[key] !== 'undefined') {
+          prev[key] = env[key]!
+        } else {
+          missingEnvKeys.add(key)
+        }
+        return prev
+      }, {})
+    : {}
+
+  if (missingEnvKeys.size > 0) {
+    console.warn(
+      `Missing env value${missingEnvKeys.size === 1 ? '' : 's'}: ${[
+        ...missingEnvKeys,
+      ].join(', ')} for ${page}.\n` +
+        `Make sure to supply this value in either your .env file or in your environment.\n` +
+        `See here for more info: https://err.sh/next.js/missing-env-value`
+    )
+  }
+  return collected
+}