feat(schema-files-loader): Customize file loading logic (#24085)

* feat(schema-files-loader): Customize file loading logic

Allows to swap files resolution logic used by the package.
By default, package still uses real fs (see `realFsResolver`).
Additinal implementatins are:

- `InMemoryFilesResolver` as the name suggests, keeps virtual file tree
in memory.
- `CompsoiteFilesResolver` combines two resolvers together.

Both will be used for language-tools, where some of the source files
might diverge from their on-disk content while editing.

Contribute to prisma/team-orm#1042

* Windows, why are you like this

* Build correct file

* Handle case-sensitivity

* Check preview feature when looking up related files

* Fix formatting

* Finish writing comment

* Split paths by either slash

* Unbreak lockfile

* Split by path better

packages/schema-files-loader/helpers/build.ts

@@ -3,9 +3,9 @@ import { build } from '../../../helpers/compile/build'
 void build([
   {
     name: 'default',
-    entryPoints: ['src/schema-files-loader.ts'],
+    entryPoints: ['src/index.ts'],
     outfile: 'dist/index',
-    external: ['fs-extra'],
+    external: ['fs-extra', '@prisma/prisma-schema-wasm'],
     bundle: true,
     emitTypes: true,
   },

packages/schema-files-loader/package.json

@@ -22,6 +22,7 @@
   ],
   "sideEffects": false,
   "dependencies": {
+    "@prisma/prisma-schema-wasm": "5.14.0-17.56ca112d5a19c9925b53af75c3c6b7ada97f9f85",
     "fs-extra": "11.1.1"
   },
   "devDependencies": {

packages/schema-files-loader/src/__fixtures__/related-feature/a.prisma

@@ -0,0 +1,4 @@
+generator client {
+    provider        = "prisma-client-js"
+    previewFeatures = ["prismaSchemaFolder"]
+}

packages/schema-files-loader/src/__fixtures__/related-feature/b.prisma

@@ -0,0 +1 @@
+// this is b

packages/schema-files-loader/src/__fixtures__/related-no-feature/a.prisma

@@ -0,0 +1,3 @@
+generator client {
+    provider = "prisma-client-js"
+}

packages/schema-files-loader/src/__fixtures__/related-no-feature/b.prisma

@@ -0,0 +1 @@
+// this is b

packages/schema-files-loader/src/index.ts

@@ -1 +1,4 @@
-export * from './schema-files-loader'
+export * from './loadRelatedSchemaFiles'
+export * from './loadSchemaFiles'
+export * from './resolver'
+export * from './usesPrismaSchemaFolder'

packages/schema-files-loader/src/loadRelatedSchemaFiles.test.ts

@@ -0,0 +1,12 @@
+import { loadRelatedSchemaFiles } from './loadRelatedSchemaFiles'
+import { fixturePath, loadedFile } from './testUtils'
+
+test('without feature enabled', async () => {
+  const files = await loadRelatedSchemaFiles(fixturePath('related-no-feature', 'a.prisma'))
+  expect(files).toEqual([loadedFile('related-no-feature', 'a.prisma')])
+})
+
+test('with feature enabled', async () => {
+  const files = await loadRelatedSchemaFiles(fixturePath('related-feature', 'a.prisma'))
+  expect(files).toEqual([loadedFile('related-feature', 'a.prisma'), loadedFile('related-feature', 'b.prisma')])
+})

packages/schema-files-loader/src/loadRelatedSchemaFiles.ts

@@ -0,0 +1,46 @@
+import path from 'node:path'
+
+import { get_config } from '@prisma/prisma-schema-wasm'
+
+import { LoadedFile, loadSchemaFiles } from './loadSchemaFiles'
+import { FilesResolver, realFsResolver } from './resolver'
+import { ConfigMetaFormat, usesPrismaSchemaFolder } from './usesPrismaSchemaFolder'
+
+/**
+ * Given a single file path, returns
+ * all files composing the same schema
+ * @param filePath
+ * @param filesResolver
+ * @returns
+ */
+export async function loadRelatedSchemaFiles(
+  filePath: string,
+  filesResolver: FilesResolver = realFsResolver,
+): Promise<LoadedFile[]> {
+  const files = await loadSchemaFiles(path.dirname(filePath), filesResolver)
+  if (isPrismaFolderEnabled(files)) {
+    return files
+  }
+  // if feature is not enabled, return only supplied file
+  const contents = await filesResolver.getFileContents(filePath)
+  if (!contents) {
+    return []
+  }
+  return [[filePath, contents]]
+}
+
+function isPrismaFolderEnabled(files: LoadedFile[]): boolean {
+  const params = JSON.stringify({
+    prismaSchema: files,
+    datasourceOverrides: {},
+    ignoreEnvVarErrors: true,
+    env: {},
+  })
+
+  try {
+    const response = JSON.parse(get_config(params)) as ConfigMetaFormat
+    return usesPrismaSchemaFolder(response)
+  } catch (e) {
+    return false
+  }
+}

packages/schema-files-loader/src/loadSchemaFiles.test.ts

@@ -0,0 +1,69 @@
+import path from 'node:path'
+
+import { loadSchemaFiles } from './loadSchemaFiles'
+import { CompositeFilesResolver, InMemoryFilesResolver, realFsResolver } from './resolver'
+import { fixturePath, line } from './testUtils'
+
+test('simple list', async () => {
+  const files = await loadSchemaFiles(fixturePath('simple'))
+  expect(files).toEqual([
+    [fixturePath('simple', 'a.prisma'), line('// this is a')],
+    [fixturePath('simple', 'b.prisma'), line('// this is b')],
+  ])
+})
+
+test('ignores non *.prisma files', async () => {
+  const files = await loadSchemaFiles(fixturePath('non-prisma-files'))
+  expect(files).toEqual([
+    [fixturePath('non-prisma-files', 'a.prisma'), line('// this is a')],
+    [fixturePath('non-prisma-files', 'b.prisma'), line('// this is b')],
+  ])
+})
+
+test('ignores subfolders *.prisma files', async () => {
+  const files = await loadSchemaFiles(fixturePath('subfolder'))
+  expect(files).toEqual([[fixturePath('subfolder', 'a.prisma'), line('// this is a')]])
+})
+
+test('ignores *.prisma directories', async () => {
+  const files = await loadSchemaFiles(fixturePath('with-directory'))
+  expect(files).toEqual([[fixturePath('with-directory', 'a.prisma'), line('// this is a')]])
+})
+
+test('reads symlinks', async () => {
+  const files = await loadSchemaFiles(fixturePath('symlink'))
+  // link point to `simple` directory
+  expect(files).toEqual([[fixturePath('simple', 'a.prisma'), line('// this is a')]])
+})
+
+test('ignores symlinks to directories', async () => {
+  const files = await loadSchemaFiles(fixturePath('symlinks-to-dir'))
+  expect(files).toEqual([])
+})
+
+test('allows to use in-memory resolver', async () => {
+  const resolver = new InMemoryFilesResolver({ caseSensitive: true })
+  resolver.addFile(path.join('/', 'some', 'dir', 'a.prisma'), '// this is a')
+  resolver.addFile(path.join('/', 'some', 'dir', 'b.prisma'), '// this is b')
+  const files = await loadSchemaFiles('/some/dir', resolver)
+
+  expect(files).toEqual([
+    [path.join('/', 'some', 'dir', 'a.prisma'), '// this is a'],
+    [path.join('/', 'some', 'dir', 'b.prisma'), '// this is b'],
+  ])
+})
+
+test('allows to use composite resolver', async () => {
+  const inMemory = new InMemoryFilesResolver({ caseSensitive: true })
+  inMemory.addFile(fixturePath('simple', 'b.prisma'), line('// b is overridden'))
+  inMemory.addFile(fixturePath('simple', 'c.prisma'), line('// in memory only'))
+
+  const resolver = new CompositeFilesResolver(inMemory, realFsResolver, { caseSensitive: true })
+  const files = await loadSchemaFiles(fixturePath('simple'), resolver)
+
+  expect(files).toEqual([
+    [fixturePath('simple', 'b.prisma'), line('// b is overridden')],
+    [fixturePath('simple', 'c.prisma'), line('// in memory only')],
+    [fixturePath('simple', 'a.prisma'), line('// this is a')],
+  ])
+})

packages/schema-files-loader/src/loadSchemaFiles.ts

@@ -0,0 +1,58 @@
+import path from 'node:path'
+
+import { FilesResolver, realFsResolver } from './resolver'
+
+export type LoadedFile = [filePath: string, content: string]
+
+type InternalValidatedEntry =
+  | {
+      valid: false
+    }
+  | {
+      valid: true
+      fullPath: string
+      content: string
+    }
+
+/**
+ * Given folder name, returns list of all files composing a single prisma schema
+ * @param folderPath
+ */
+export async function loadSchemaFiles(
+  folderPath: string,
+  filesResolver: FilesResolver = realFsResolver,
+): Promise<LoadedFile[]> {
+  const dirEntries = await filesResolver.listDirContents(folderPath)
+  const validatedList = await Promise.all(
+    dirEntries.map((filePath) => validateFilePath(path.join(folderPath, filePath), filesResolver)),
+  )
+  return validatedList.reduce((acc, entry) => {
+    if (entry.valid) {
+      acc.push([entry.fullPath, entry.content])
+    }
+    return acc
+  }, [] as LoadedFile[])
+}
+
+async function validateFilePath(fullPath: string, filesResolver: FilesResolver): Promise<InternalValidatedEntry> {
+  if (path.extname(fullPath) !== '.prisma') {
+    return { valid: false }
+  }
+  const fileType = await filesResolver.getEntryType(fullPath)
+
+  if (!fileType) {
+    return { valid: false }
+  }
+  if (fileType.kind === 'file') {
+    const content = await filesResolver.getFileContents(fullPath)
+    if (typeof content === 'undefined') {
+      return { valid: false }
+    }
+    return { valid: true, fullPath, content }
+  }
+  if (fileType.kind === 'symlink') {
+    const realPath = fileType.realPath
+    return validateFilePath(realPath, filesResolver)
+  }
+  return { valid: false }
+}

packages/schema-files-loader/src/resolver/CompositeFilesResolver.test.ts

@@ -0,0 +1,114 @@
+import { CompositeFilesResolver } from './CompositeFilesResolver'
+import { InMemoryFilesResolver } from './InMemoryFilesResolver'
+
+describe('caseSensitive=true', () => {
+  test('listDirContents', async () => {
+    const primary = new InMemoryFilesResolver({ caseSensitive: true })
+    primary.addFile('/dir/in-primary.prisma', '// a')
+    primary.addFile('/dir/different-case.prisma', '// different case primary')
+    primary.addFile('/dir/in-both.prisma', '// b')
+
+    const secondary = new InMemoryFilesResolver({ caseSensitive: true })
+    secondary.addFile('/dir/in-both.prisma', '// b old')
+    secondary.addFile('/dir/Different-Case.prisma', '// different case secondary')
+    secondary.addFile('/dir/in-secondary.prisma', '// c')
+
+    const composite = new CompositeFilesResolver(primary, secondary, { caseSensitive: true })
+    const contents = await composite.listDirContents('/dir')
+
+    expect(contents).toEqual([
+      'in-primary.prisma',
+      'different-case.prisma',
+      'in-both.prisma',
+      'Different-Case.prisma',
+      'in-secondary.prisma',
+    ])
+  })
+
+  test('getEntryType', async () => {
+    const primary = new InMemoryFilesResolver({ caseSensitive: true })
+    primary.addFile('/dir/a.prisma', '// a')
+    primary.addFile('/dir/dir1/.thing', '')
+
+    const secondary = new InMemoryFilesResolver({ caseSensitive: true })
+    secondary.addFile('/dir/b.prisma', '// b')
+    secondary.addFile('/dir/dir1', '// dir 1')
+
+    const composite = new CompositeFilesResolver(primary, secondary, { caseSensitive: true })
+
+    expect(await composite.getEntryType('/dir/a.prisma')).toEqual({ kind: 'file' })
+    expect(await composite.getEntryType('/dir/A.prisma')).toBeUndefined()
+    expect(await composite.getEntryType('/dir/dir1')).toEqual({ kind: 'directory' })
+    expect(await composite.getEntryType('/dir/does-not-exist')).toBeUndefined()
+  })
+
+  test('getFileContents', async () => {
+    const primary = new InMemoryFilesResolver({ caseSensitive: true })
+    primary.addFile('/dir/in-primary.prisma', '// from primary')
+    primary.addFile('/dir/in-both.prisma', '// primary overrides secondary')
+
+    const secondary = new InMemoryFilesResolver({ caseSensitive: true })
+    secondary.addFile('/dir/in-both.prisma', '// this should not be returned')
+    secondary.addFile('/dir/in-secondary.prisma', '// from secondary')
+
+    const composite = new CompositeFilesResolver(primary, secondary, { caseSensitive: true })
+
+    expect(await composite.getFileContents('/dir/in-primary.prisma')).toBe('// from primary')
+    expect(await composite.getFileContents('/dir/IN-PRIMARY.prisma')).toBeUndefined()
+    expect(await composite.getFileContents('/dir/in-both.prisma')).toBe('// primary overrides secondary')
+    expect(await composite.getFileContents('/dir/in-secondary.prisma')).toBe('// from secondary')
+  })
+})
+
+describe('caseSensitive=false', () => {
+  test('listDirContents', async () => {
+    const primary = new InMemoryFilesResolver({ caseSensitive: false })
+    primary.addFile('/dir/in-primary.prisma', '// a')
+    primary.addFile('/dir/different-case.prisma', '// different case primary')
+    primary.addFile('/dir/in-both.prisma', '// b')
+
+    const secondary = new InMemoryFilesResolver({ caseSensitive: false })
+    secondary.addFile('/dir/in-both.prisma', '// b old')
+    secondary.addFile('/dir/Different-Case.prisma', '// different case secondary')
+    secondary.addFile('/dir/in-secondary.prisma', '// c')
+
+    const composite = new CompositeFilesResolver(primary, secondary, { caseSensitive: false })
+    const contents = await composite.listDirContents('/dir')
+
+    expect(contents).toEqual(['in-primary.prisma', 'different-case.prisma', 'in-both.prisma', 'in-secondary.prisma'])
+  })
+
+  test('getEntryType', async () => {
+    const primary = new InMemoryFilesResolver({ caseSensitive: false })
+    primary.addFile('/dir/a.prisma', '// a')
+    primary.addFile('/dir/dir1/.thing', '')
+
+    const secondary = new InMemoryFilesResolver({ caseSensitive: false })
+    secondary.addFile('/dir/b.prisma', '// b')
+    secondary.addFile('/dir/dir1', '// dir 1')
+
+    const composite = new CompositeFilesResolver(primary, secondary, { caseSensitive: false })
+
+    expect(await composite.getEntryType('/dir/a.prisma')).toEqual({ kind: 'file' })
+    expect(await composite.getEntryType('/dir/A.prisma')).toEqual({ kind: 'file' })
+    expect(await composite.getEntryType('/dir/dir1')).toEqual({ kind: 'directory' })
+    expect(await composite.getEntryType('/dir/does-not-exist')).toBeUndefined()
+  })
+
+  test('getFileContents', async () => {
+    const primary = new InMemoryFilesResolver({ caseSensitive: false })
+    primary.addFile('/dir/in-primary.prisma', '// from primary')
+    primary.addFile('/dir/in-both.prisma', '// primary overrides secondary')
+
+    const secondary = new InMemoryFilesResolver({ caseSensitive: false })
+    secondary.addFile('/dir/in-both.prisma', '// this should not be returned')
+    secondary.addFile('/dir/in-secondary.prisma', '// from secondary')
+
+    const composite = new CompositeFilesResolver(primary, secondary, { caseSensitive: false })
+
+    expect(await composite.getFileContents('/dir/in-primary.prisma')).toBe('// from primary')
+    expect(await composite.getFileContents('/dir/IN-PRIMARY.prisma')).toBe('// from primary')
+    expect(await composite.getFileContents('/dir/in-both.prisma')).toBe('// primary overrides secondary')
+    expect(await composite.getFileContents('/dir/in-secondary.prisma')).toBe('// from secondary')
+  })
+})

packages/schema-files-loader/src/resolver/CompositeFilesResolver.ts

@@ -0,0 +1,41 @@
+import { createFileNameToKeyMapper, FileNameToKeyMapper } from './caseSensitivity'
+import { CaseSensitivityOptions, FilesResolver, FsEntryType } from './types'
+
+/**
+ * Files resolver that combines two other resolvers
+ * together. Files existing in either one of those will be
+ * reported. If content exist in both, primary resolver takes
+ * precedence
+ */
+export class CompositeFilesResolver implements FilesResolver {
+  private _fileNameToKey: FileNameToKeyMapper
+  constructor(private primary: FilesResolver, private secondary: FilesResolver, options: CaseSensitivityOptions) {
+    this._fileNameToKey = createFileNameToKeyMapper(options)
+  }
+
+  async listDirContents(path: string): Promise<string[]> {
+    const primaryContent = await this.primary.listDirContents(path)
+    const secondaryContent = await this.secondary.listDirContents(path)
+
+    return uniqueWith([...primaryContent, ...secondaryContent], this._fileNameToKey)
+  }
+
+  async getEntryType(path: string): Promise<FsEntryType | undefined> {
+    return (await this.primary.getEntryType(path)) ?? (await this.secondary.getEntryType(path))
+  }
+
+  async getFileContents(path: string): Promise<string | undefined> {
+    return (await this.primary.getFileContents(path)) ?? (await this.secondary.getFileContents(path))
+  }
+}
+
+function uniqueWith(fileNames: string[], toKey: FileNameToKeyMapper): string[] {
+  const map = new Map<string, string>()
+  for (const fileName of fileNames) {
+    const key = toKey(fileName)
+    if (!map.has(key)) {
+      map.set(key, fileName)
+    }
+  }
+  return Array.from(map.values())
+}