Skip to content

Commit

Permalink
fix(csv-parse): encoding detection with bom (#350)
Browse files Browse the repository at this point in the history
  • Loading branch information
@moroine
moroine committed Jul 1, 2022
1 parent e8b4fc7 commit fd75e66
Show file tree
Hide file tree
Showing 10 changed files with 37 additions and 19 deletions.
6 changes: 4 additions & 2 deletions packages/csv-parse/dist/cjs/index.cjs
View file
Expand Up @@ -635,8 +635,8 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {record_delimiter} = this.options;
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter, } = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
if(previousBuf === undefined){
Expand Down Expand Up @@ -672,6 +672,8 @@ const transform = function(original_options = {}) {
buf = buf.slice(bomLength);
// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
break;
}
}
Expand Down
6 changes: 4 additions & 2 deletions packages/csv-parse/dist/cjs/sync.cjs
View file
Expand Up @@ -633,8 +633,8 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {record_delimiter} = this.options;
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter, } = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
if(previousBuf === undefined){
Expand Down Expand Up @@ -670,6 +670,8 @@ const transform = function(original_options = {}) {
buf = buf.slice(bomLength);
// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
break;
}
}
Expand Down
6 changes: 4 additions & 2 deletions packages/csv-parse/dist/esm/index.js
View file
Expand Up @@ -5691,8 +5691,8 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {record_delimiter} = this.options;
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter, } = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
if(previousBuf === undefined){
Expand Down Expand Up @@ -5728,6 +5728,8 @@ const transform = function(original_options = {}) {
buf = buf.slice(bomLength);
// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
break;
}
}
Expand Down
6 changes: 4 additions & 2 deletions packages/csv-parse/dist/esm/sync.js
View file
Expand Up @@ -2601,8 +2601,8 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {record_delimiter} = this.options;
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter, } = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
if(previousBuf === undefined){
Expand Down Expand Up @@ -2638,6 +2638,8 @@ const transform = function(original_options = {}) {
buf = buf.slice(bomLength);
// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
break;
}
}
Expand Down
6 changes: 4 additions & 2 deletions packages/csv-parse/dist/iife/index.js
View file
Expand Up @@ -5694,8 +5694,8 @@ var csv_parse = (function (exports) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {record_delimiter} = this.options;
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter, } = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
if(previousBuf === undefined){
Expand Down Expand Up @@ -5731,6 +5731,8 @@ var csv_parse = (function (exports) {
buf = buf.slice(bomLength);
// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
break;
}
}
Expand Down
6 changes: 4 additions & 2 deletions packages/csv-parse/dist/iife/sync.js
View file
Expand Up @@ -2604,8 +2604,8 @@ var csv_parse_sync = (function (exports) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {record_delimiter} = this.options;
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter, } = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
if(previousBuf === undefined){
Expand Down Expand Up @@ -2641,6 +2641,8 @@ var csv_parse_sync = (function (exports) {
buf = buf.slice(bomLength);
// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
break;
}
}
Expand Down
6 changes: 4 additions & 2 deletions packages/csv-parse/dist/umd/index.js
View file
Expand Up @@ -5697,8 +5697,8 @@
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {record_delimiter} = this.options;
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter, } = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
if(previousBuf === undefined){
Expand Down Expand Up @@ -5734,6 +5734,8 @@
buf = buf.slice(bomLength);
// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
break;
}
}
Expand Down
6 changes: 4 additions & 2 deletions packages/csv-parse/dist/umd/sync.js
View file
Expand Up @@ -2607,8 +2607,8 @@
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {record_delimiter} = this.options;
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter, } = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
if(previousBuf === undefined){
Expand Down Expand Up @@ -2644,6 +2644,8 @@
buf = buf.slice(bomLength);
// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
break;
}
}
Expand Down
6 changes: 4 additions & 2 deletions packages/csv-parse/lib/api/index.js
View file
Expand Up @@ -55,8 +55,8 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {record_delimiter} = this.options;
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter, } = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
if(previousBuf === undefined){
Expand Down Expand Up @@ -92,6 +92,8 @@ const transform = function(original_options = {}) {
buf = buf.slice(bomLength);
// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
break;
}
}
Expand Down
2 changes: 1 addition & 1 deletion packages/csv-parse/test/option.encoding.coffee
View file
Expand Up @@ -78,6 +78,6 @@ describe 'Option `encoding`', ->
] unless err
next err
# parser.write Buffer.from Buffer.from([255, 254])
parser.write Buffer.from "\ufeffa,b,c\n", 'utf16le'
parser.write Buffer.from '\ufeffa,b,"c"\n', 'utf16le'
parser.write Buffer.from 'd,e,f', 'utf16le'
parser.end()

0 comments on commit fd75e66

Please sign in to comment.