diff --git a/docs/content/en/configuration.md b/docs/content/en/configuration.md index a2b91f776..3a5fbd731 100644 --- a/docs/content/en/configuration.md +++ b/docs/content/en/configuration.md @@ -524,6 +524,13 @@ Your component should implement the following: You should be aware that you get the full markdown file content so this includes the front-matter. You can use `gray-matter` to split and join the markdown and the front-matter. +### `useCache` + +- Type: `Boolean` +- Default: `false` + +When `true`, the production server (`nuxt start`) will use cached version of the content (generated after running `nuxt build`) instead of parsing files. This improves app startup time, but makes app unaware of any content changes. + ## Defaults ```js{}[nuxt.config.js] @@ -535,6 +542,7 @@ export default { fullTextSearchFields: ['title', 'description', 'slug', 'text'], nestedProperties: [], liveEdit: true, + useCache: false, markdown: { remarkPlugins: [ 'remark-squeeze-paragraphs', diff --git a/packages/content/lib/database.js b/packages/content/lib/database.js index 269feb499..df80a8e12 100644 --- a/packages/content/lib/database.js +++ b/packages/content/lib/database.js @@ -1,5 +1,6 @@ const { join, extname } = require('path') const fs = require('graceful-fs').promises +const mkdirp = require('mkdirp') const Hookable = require('hookable') const chokidar = require('chokidar') const JSON5 = require('json5') @@ -18,7 +19,9 @@ class Database extends Hookable { constructor (options) { super() this.dir = options.dir || process.cwd() - this.cwd = options.cwd || process.cwd() + this.srcDir = options.srcDir || process.cwd() + this.buildDir = options.buildDir || process.cwd() + this.useCache = options.useCache || false this.markdown = new Markdown(options.markdown) this.yaml = new YAML(options.yaml) this.csv = new CSV(options.csv) @@ -26,10 +29,11 @@ class Database extends Hookable { // Create Loki database this.db = new Loki('content.db') // Init collection - this.items = this.db.addCollection('items', { + this.itemsCollectionOptions = { fullTextSearch: options.fullTextSearchFields.map(field => ({ field })), nestedProperties: options.nestedProperties - }) + } + this.items = this.db.addCollection('items', this.itemsCollectionOptions) // User Parsers this.extendParser = options.extendParser || {} this.extendParserExtensions = Object.keys(this.extendParser) @@ -58,19 +62,65 @@ class Database extends Hookable { }, this.options) } + async init () { + if (this.useCache) { + try { + return await this.initFromCache() + } catch (error) {} + } + + await this.initFromFilesystem() + } + /** * Clear items in database and load files into collection */ - async init () { + async initFromFilesystem () { + const startTime = process.hrtime() this.dirs = ['/'] this.items.clear() - - const startTime = process.hrtime() await this.walk(this.dir) const [s, ns] = process.hrtime(startTime) logger.info(`Parsed ${this.items.count()} files in ${s}.${Math.round(ns / 1e8)} seconds`) } + async initFromCache () { + const startTime = process.hrtime() + const cacheFilePath = join(this.buildDir, this.db.filename) + const cacheFileData = await fs.readFile(cacheFilePath, 'utf-8') + const cacheFileJson = JSON.parse(cacheFileData) + + this.db.loadJSONObject(cacheFileJson) + + // recreate references + this.items = this.db.getCollection('items') + this.dirs = this.items.mapReduce(doc => doc.dir, dirs => [...new Set(dirs)]) + + const [s, ns] = process.hrtime(startTime) + logger.info(`Loaded ${this.items.count()} documents from cache in ${s},${Math.round(ns / 1e8)} seconds`) + } + + /** + * Store database info file + * @param {string} [dir] - Directory containing database dump file. + * @param {string} [filename] - Database dump filename. + */ + async save (dir, filename) { + dir = dir || this.buildDir + filename = filename || this.db.filename + + await mkdirp(dir) + await fs.writeFile(join(dir, filename), this.db.serialize(), 'utf-8') + } + + async rebuildCache () { + logger.info('Rebuilding content cache') + this.db = new Loki('content.db') + this.items = this.db.addCollection('items', this.itemsCollectionOptions) + await this.initFromFilesystem() + await this.save() + } + /** * Walk dir tree recursively * @param {string} dir - Directory to browse. @@ -145,7 +195,7 @@ class Database extends Hookable { const document = this.items.findOne({ path: item.path }) - logger.info(`Updated ${path.replace(this.cwd, '.')}`) + logger.info(`Updated ${path.replace(this.srcDir, '.')}`) if (document) { this.items.update({ $loki: document.$loki, meta: document.meta, ...item }) return @@ -171,7 +221,7 @@ class Database extends Hookable { */ async parseFile (path) { const extension = extname(path) - // If unkown extension, skip + // If unknown extension, skip if (!EXTENSIONS.includes(extension) && !this.extendParserExtensions.includes(extension)) { return } @@ -204,7 +254,7 @@ class Database extends Hookable { // Force data to be an array data = Array.isArray(data) ? data : [data] } catch (err) { - logger.warn(`Could not parse ${path.replace(this.cwd, '.')}:`, err.message) + logger.warn(`Could not parse ${path.replace(this.srcDir, '.')}:`, err.message) return null } diff --git a/packages/content/lib/index.js b/packages/content/lib/index.js index ab10e6ab9..b26d718c2 100644 --- a/packages/content/lib/index.js +++ b/packages/content/lib/index.js @@ -1,6 +1,5 @@ const { join, resolve } = require('path') const fs = require('graceful-fs').promises -const mkdirp = require('mkdirp') const defu = require('defu') const logger = require('consola').withScope('@nuxt/content') const hash = require('hasha') @@ -95,11 +94,21 @@ module.exports = async function (moduleOptions) { server.on('upgrade', (...args) => ws.callHook('upgrade', ...args)) }) + const useCache = options.useCache && !this.options.dev && this.options.ssr + const database = new Database({ ...options, - cwd: this.options.srcDir + srcDir: this.options.srcDir, + buildDir: resolve(this.options.buildDir, 'content'), + useCache }) + if (useCache) { + this.nuxt.hook('builder:prepared', async () => { + await database.rebuildCache() + }) + } + // Database hooks database.hook('file:beforeInsert', item => this.nuxt.callHook('content:file:beforeInsert', item, database) @@ -187,12 +196,7 @@ module.exports = async function (moduleOptions) { this.nuxt.hook('generate:distRemoved', async () => { const dir = resolve(this.options.buildDir, 'dist', 'client', 'content') - await mkdirp(dir) - await fs.writeFile( - join(dir, `db-${dbHash}.json`), - database.db.serialize(), - 'utf-8' - ) + await database.save(dir, `db-${dbHash}.json`) }) // Add client plugin diff --git a/packages/content/lib/utils.js b/packages/content/lib/utils.js index ed0200b97..aea36ab29 100644 --- a/packages/content/lib/utils.js +++ b/packages/content/lib/utils.js @@ -4,6 +4,7 @@ const { camelCase } = require('change-case') const getDefaults = ({ dev = false } = {}) => ({ editor: './editor.vue', watch: dev, + useCache: false, liveEdit: true, apiPrefix: '_content', dir: 'content', diff --git a/packages/content/test/cache.test.js b/packages/content/test/cache.test.js new file mode 100644 index 000000000..5d915da13 --- /dev/null +++ b/packages/content/test/cache.test.js @@ -0,0 +1,62 @@ +const path = require('path') +const fs = require('graceful-fs').promises +const { build, init, generatePort, loadConfig } = require('@nuxtjs/module-test-utils') + +describe('content cache', () => { + const config = { + ...loadConfig(__dirname), + buildDir: path.join(__dirname, 'fixture', '.nuxt-dev'), + content: { + useCache: true + } + } + + const dbFilePath = path.join(config.buildDir, 'content', 'content.db') + + describe('during build', () => { + let nuxt + + beforeAll(async () => { + fs.unlink(dbFilePath).catch(() => {}); + ({ nuxt } = (await build(config))) + }, 60000) + + afterAll(async () => { + await nuxt.close() + }) + + test('should be generated', async () => { + await expect(fs.access(dbFilePath)).resolves.not.toThrow() + }) + + test('should be a valid json', async () => { + const fileTextContent = await fs.readFile(dbFilePath) + await expect(() => JSON.parse(fileTextContent)).not.toThrow() + }) + }) + + describe('during start in production mode', () => { + const mockDbDump = '{"_env":"NODEJS","_serializationMethod":"normal","_autosave":false,"_autosaveInterval":5000,"_collections":[{"name":"items","unindexedSortComparator":"js","defaultLokiOperatorPackage":"js","_dynamicViews":[],"uniqueNames":[],"transforms":{},"rangedIndexes":{},"_data":[{"slug":"about","title":"Serialized test","toc":[],"body":{"type":"root","children":[{"type":"element","tag":"p","props":{},"children":[{"type":"text","value":"This is the serialized page!"}]}]},"text":"\\nThis is the serialized page!\\n","dir":"/","path":"/about","extension":".md","createdAt":"2021-02-11T22:10:21.655Z","updatedAt":"2021-02-12T20:13:23.079Z","meta":{"version":0,"revision":0,"created":1613160831274},"$loki":1}],"idIndex":[1],"maxId":1,"_dirty":true,"_nestedProperties":[],"transactional":false,"asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"cloneObjects":false,"cloneMethod":"deep","changes":[],"_fullTextSearch":{"ii":{"title":{"_store":true,"_optimizeChanges":true,"docCount":1,"docStore":[[0,{"fieldLength":2}]],"totalFieldLength":2,"root":{"k":[115,116],"v":[{"k":[101],"v":[{"k":[114],"v":[{"k":[105],"v":[{"k":[97],"v":[{"k":[108],"v":[{"k":[105],"v":[{"k":[122],"v":[{"k":[101],"v":[{"k":[100],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}]}]}]}]}]},{"k":[101],"v":[{"k":[115],"v":[{"k":[116],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}},"description":{"_store":true,"_optimizeChanges":true,"docCount":0,"docStore":[],"totalFieldLength":0,"root":{}},"slug":{"_store":true,"_optimizeChanges":true,"docCount":1,"docStore":[[0,{"fieldLength":1}]],"totalFieldLength":1,"root":{"k":[97],"v":[{"k":[98],"v":[{"k":[111],"v":[{"k":[117],"v":[{"k":[116],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}]}},"text":{"_store":true,"_optimizeChanges":true,"docCount":1,"docStore":[[0,{"fieldLength":5}]],"totalFieldLength":5,"root":{"k":[116,105,115,112],"v":[{"k":[104],"v":[{"k":[105,101],"v":[{"k":[115],"v":[{"d":{"df":1,"dc":[[0,1]]}}]},{"d":{"df":1,"dc":[[0,1]]}}]}]},{"k":[115],"v":[{"d":{"df":1,"dc":[[0,1]]}}]},{"k":[101],"v":[{"k":[114],"v":[{"k":[105],"v":[{"k":[97],"v":[{"k":[108],"v":[{"k":[105],"v":[{"k":[122],"v":[{"k":[101],"v":[{"k":[100],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}]}]}]}]}]},{"k":[97],"v":[{"k":[103],"v":[{"k":[101],"v":[{"k":[33],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}]}}}}}],"databaseVersion":1.5,"engineVersion":1.5,"filename":"content.db","_persistenceAdapter":null,"_persistenceMethod":null,"_throttledSaves":true}' + let nuxt + let $content + + beforeAll(async () => { + await fs.writeFile(dbFilePath, mockDbDump) + nuxt = await init(config) + await nuxt.listen(await generatePort()) + $content = require('@nuxt/content').$content + }, 60000) + + afterAll(async () => { + await nuxt.close() + }) + + test('should use cached db', async () => { + const item = await $content('about').fetch() + + expect(item).toEqual(expect.objectContaining({ + title: 'Serialized test' + })) + }) + }) +}) diff --git a/packages/content/test/options.test.js b/packages/content/test/options.test.js index 93424fe15..9d6483bb4 100644 --- a/packages/content/test/options.test.js +++ b/packages/content/test/options.test.js @@ -32,6 +32,7 @@ describe('options', () => { expect(options).toEqual(expect.objectContaining({ apiPrefix: '_content', dir: 'content', + useCache: false, fullTextSearchFields: ['title', 'description', 'slug', 'text'], nestedProperties: [], csv: {},