How to import flatgeobuf in nodejs env?

[jackcjp]

When I import in this way:
const geojson = require('flatgeobuf/lib/cjs/flatgeobuf.js');

it throw error:

node:internal/modules/cjs/loader:488
      throw e;
      ^

Error [ERR_PACKAGE_PATH_NOT_EXPORTED]: Package subpath './lib/cjs/flatgeobuf.js' is not defined by "exports" in /mnt/d/workspace/flatgeobuf_test/node_modules/flatgeobuf/package.json
    at new NodeError (node:internal/errors:371:5)
    at throwExportsNotFound (node:internal/modules/esm/resolve:440:9)
    at packageExportsResolve (node:internal/modules/esm/resolve:692:3)
    at resolveExports (node:internal/modules/cjs/loader:482:36)
    at Function.Module._findPath (node:internal/modules/cjs/loader:522:31)
    at Function.Module._resolveFilename (node:internal/modules/cjs/loader:919:27)
    at Function.Module._load (node:internal/modules/cjs/loader:778:27)
    at Module.require (node:internal/modules/cjs/loader:1005:19)
    at require (node:internal/modules/cjs/helpers:102:18)
    at Object.<anonymous> (/mnt/d/workspace/flatgeobuf_test/mjs/server.cjs:5:17) {
  code: 'ERR_PACKAGE_PATH_NOT_EXPORTED'
}

Environment:
Node: v16.13.0
npm: 8.1.4
os: Ubuntu-20.04
"flatgeobuf": "^3.20.1",
"node-fetch": "^2.6.6"

const resource = 'https://gvpub.oss-cn-beijing.aliyuncs.com/UScounties.fgb';
const http = require('http');
const url = require('url');
const fetch  = require('node-fetch');
const geojson = require('flatgeobuf/lib/cjs/flatgeobuf.js');
const port = 3000;
let server = http.createServer(async (req, res) => {
    const urlObj = url.parse(req.url || '', true);
    const { pathname, query } = urlObj;
    if (pathname && pathname.startsWith('/api') && pathname === '/api/data') {
        if (query.bounds) {
        }
        else {
            return await getData();
        }
    }
    else
        res.write('test');
    res.end();
});
server.listen(port, () => {
    console.log(`Server is running on port: ${port}`);
});
async function getData() {
    const response = await fetch(resource, {});
    const features = [];
    console.log(response.body);
    const aa = geojson.deserialize(response.body);
    return features;
}

When I downgrade flatgeobuf to 3.17.0, and import it as below:
const geojson = require("flatgeobuf/lib/cjs/geojson")
it throws error too:

/mnt/d/workspace/aa_del/node_modules/flatgeobuf/lib/cjs/geojson.js:13
    else if (input instanceof ReadableStream)
                              ^

ReferenceError: ReadableStream is not defined
    at Object.deserialize (/mnt/d/workspace/aa_del/node_modules/flatgeobuf/lib/cjs/geojson.js:13:31)
    at getData (/mnt/d/workspace/aa_del/server.js:29:24)
    at processTicksAndRejections (node:internal/process/task_queues:96:5)
    at async Server.<anonymous> (/mnt/d/workspace/aa_del/server.js:15:20)

Can anyone give me some hint? Thanks! @bjornharrtell

[bjornharrtell]

First of all, sorry for the state of the current version of flatgeobuf package and node usage right now. I think it's related to #141 so might be blocked by TypeScript issues but I'll see what I can do.

As for use of 3.17.0 I cannot reproduce your problem, I can run the example at https://github.com/flatgeobuf/flatgeobuf/tree/master/examples/node without issues. Can you try that?

[jackcjp]

@bjornharrtell thank you for your quick reply, the example at https://github.com/flatgeobuf/flatgeobuf/tree/master/examples/node works well.

But I want to try the code at
https://observablehq.com/@bjornharrtell/streaming-flatgeobuf in node envrionment.

Here is my sample code: server.js, the nodejs and os environment are both same as above:

// const resource = 'https://gvpub.oss-cn-beijing.aliyuncs.com/UScounties.fgb';
const http = require('http');
const url = require('url');
const fetch  = require('node-fetch');
const geojson = require("flatgeobuf/lib/cjs/geojson")

const port = 3000;
let server = http.createServer(async (req, res) => {
    const urlObj = url.parse(req.url || '', true);
    const { pathname, query } = urlObj;
    if (pathname && pathname.startsWith('/api') && pathname === '/api/data') {
        res.write(JSON.stringify(await getData()));
    }
    else
        res.write('test');
    res.end();
});
server.listen(port, () => {
    console.log(`Server is running on port: ${port}`);
});
async function getData() {
    // const response = await fetch(resource);
    const response = await fetch('https://flatgeobuf.org/test/data/UScounties.fgb');
    const features = [];
    const aa = geojson.deserialize(response.body);
    return features;
}

when I running the with node server.js, the error is below:

/mnt/d/workspace/flatgeobuf_test/node_modules/flatgeobuf/lib/cjs/geojson.js:13
    else if (input instanceof ReadableStream)
                              ^

ReferenceError: ReadableStream is not defined
    at Object.deserialize (/mnt/d/workspace/flatgeobuf_test/node_modules/flatgeobuf/lib/cjs/geojson.js:13:31)
    at getData (/mnt/d/workspace/flatgeobuf_test/server.js:25:24)
    at processTicksAndRejections (node:internal/process/task_queues:96:5)
    at async Server.<anonymous> (/mnt/d/workspace/flatgeobuf_test/server.js:12:34)

Is it the error file format when using https://flatgeobuf.org/test/data/UScounties.fgb?

Thanks

[bjornharrtell]

@jackcjp thanks for getting back, the problem is not the file it's deserialize implementation that takes a stream that doesn't work in node because it doesn't know what ReadableStream is. I think it should be possible to get it to run under node if you use the polyfill web-streams-polyfill. I'll see if I can get the example to demonstrate that.

[bjornharrtell]

@jackcjp it also looks like native support for ReadableStream was added in Node 16. Can you upgrade to it and try that out?

[bjornharrtell]

I now see that you run with Node 16 already, so it's surprising to me that it cannot find ReadableStream. I will try to reproduce.

[jackcjp]

I import web-streams-polyfill/ponyfill as below:

var streams = require("web-streams-polyfill/ponyfill");
const response = await fetch('https://gvpub.oss-cn-beijing.aliyuncs.com/RbushTree.json');
const aa = geojson.deserialize(new streams.ReadableStream(response.body));

If this is the correct usage, it doesn't work either with the same error.

[bjornharrtell]

@jackcjp you will have do global.ReadableStream = require("web-streams-polyfill/ponyfill") to make it available globally.

[bjornharrtell]

I've experimented a bit but it won't work by simply polyfilling ReadableStream. This is because node-fetch gives a node Stream as response.body, not a ReadableStream.

[bjornharrtell]

So I don't see any way around that you need some code to convert a node stream to a ReadableStream. This works for me with flatgeobuf 3.17.0:

const geojson = require('flatgeobuf/lib/cjs/geojson')
const fetch = require('node-fetch')
const ReadableStream = require('web-streams-polyfill').ReadableStream

function nodeToWeb(nodeStream) {
    let destroyed = false;
    const listeners = {};

    function start(controller) {
        listeners['data'] = onData;
        listeners['end'] = onData;
        listeners['end'] = onDestroy;
        listeners['close'] = onDestroy;
        listeners['error'] = onDestroy;
        for (const name in listeners) nodeStream.on(name, listeners[name]);

        nodeStream.pause();

        function onData(chunk) {
            if (destroyed) return;
            controller.enqueue(chunk);
            nodeStream.pause();
        }

        function onDestroy(err) {
            if (destroyed) return;
            destroyed = true;

            for (const name in listeners)
                nodeStream.removeListener(name, listeners[name]);

            if (err) controller.error(err);
            else controller.close();
        }
    }

    function pull() {
        if (destroyed) return;
        nodeStream.resume();
    }

    function cancel() {
        destroyed = true;

        for (const name in listeners)
            nodeStream.removeListener(name, listeners[name]);

        nodeStream.push(null);
        nodeStream.pause();
        nodeStream.destroy();
    }

    return new ReadableStream({ start: start, pull: pull, cancel: cancel });
}

async function streamtest() {
    const response = await fetch('https://flatgeobuf.org/test/data/UScounties.fgb')
    for await (let feature of geojson.deserialize(nodeToWeb(response.body)))
        console.log(JSON.stringify(feature, undefined, 1))
}

streamtest()

I hope to have better examples when I can get the current release node usage situation cleared up. I think I will support ES modules import/export only.

[bjornharrtell]

Manwhile I believe I've fixed #141 but by dropping commonjs support entirely so going forward will be with Node 16+ ES modules support. I will try to find time to fixup the node example to it.

[bjornharrtell]

Node examples are now updated to use latest version and included a specific example for how to use with ReadableStream support.