Skip to content
This repository has been archived by the owner on Jun 28, 2021. It is now read-only.

Commit

Permalink
feat: new ignore_last_delimiters option, solve #193
Browse files Browse the repository at this point in the history
  • Loading branch information
wdavidw committed Jan 18, 2021
1 parent 121add5 commit 4d70b33
Show file tree
Hide file tree
Showing 8 changed files with 155 additions and 10 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Please join and contribute:

## Trunk

* feat: new ignore_last_delimiters option, solve #193
* feat: generate browser compatible lib
* refactor: rename raw to record
* docs: comment about trimable chars
Expand Down
23 changes: 22 additions & 1 deletion lib/browser/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -6744,6 +6744,24 @@ class Parser extends Transform {
throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`)
}
}
// Normalize options `ignore_last_delimiters`
if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){
options.ignore_last_delimiters = false
}else if(typeof options.ignore_last_delimiters === 'number'){
options.ignore_last_delimiters = options.ignore_last_delimiters > 0
}else if(typeof options.ignore_last_delimiters !== 'boolean'){
throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [
'Invalid option `ignore_last_delimiters`:',
'the value must be a boolean value or an integer,',
`got ${JSON.stringify(options.ignore_last_delimiters)}`
], options)
}
if(options.ignore_last_delimiters === true && options.columns === false){
throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [
'The option `ignore_last_delimiters`',
'requires the activation of the `columns` option'
], options)
}
// Normalize option `info`
if(options.info === undefined || options.info === null || options.info === false){
options.info = false
Expand Down Expand Up @@ -7533,7 +7551,10 @@ class Parser extends Transform {
return numOfCharLeft < requiredLength
}
__isDelimiter(buf, pos, chr){
const {delimiter} = this.options
const {delimiter, ignore_last_delimiters} = this.options
if(ignore_last_delimiters && this.state.record.length === this.options.columns.length - 1){
return 0
}
loop1: for(let i = 0; i < delimiter.length; i++){
const del = delimiter[i]
if(del[0] === chr){
Expand Down
23 changes: 22 additions & 1 deletion lib/browser/sync.js
Original file line number Diff line number Diff line change
Expand Up @@ -6744,6 +6744,24 @@ class Parser extends Transform {
throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`)
}
}
// Normalize options `ignore_last_delimiters`
if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){
options.ignore_last_delimiters = false
}else if(typeof options.ignore_last_delimiters === 'number'){
options.ignore_last_delimiters = options.ignore_last_delimiters > 0
}else if(typeof options.ignore_last_delimiters !== 'boolean'){
throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [
'Invalid option `ignore_last_delimiters`:',
'the value must be a boolean value or an integer,',
`got ${JSON.stringify(options.ignore_last_delimiters)}`
], options)
}
if(options.ignore_last_delimiters === true && options.columns === false){
throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [
'The option `ignore_last_delimiters`',
'requires the activation of the `columns` option'
], options)
}
// Normalize option `info`
if(options.info === undefined || options.info === null || options.info === false){
options.info = false
Expand Down Expand Up @@ -7533,7 +7551,10 @@ class Parser extends Transform {
return numOfCharLeft < requiredLength
}
__isDelimiter(buf, pos, chr){
const {delimiter} = this.options
const {delimiter, ignore_last_delimiters} = this.options
if(ignore_last_delimiters && this.state.record.length === this.options.columns.length - 1){
return 0
}
loop1: for(let i = 0; i < delimiter.length; i++){
const del = delimiter[i]
if(del[0] === chr){
Expand Down
21 changes: 20 additions & 1 deletion lib/es5/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,19 @@ var Parser = /*#__PURE__*/function (_Transform) {
} else {
throw new Error("Invalid Option: from_line must be an integer, got ".concat(JSON.stringify(opts.from_line)));
}
} // Normalize options `ignore_last_delimiters`


if (options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null) {
options.ignore_last_delimiters = false;
} else if (typeof options.ignore_last_delimiters === 'number') {
options.ignore_last_delimiters = options.ignore_last_delimiters > 0;
} else if (typeof options.ignore_last_delimiters !== 'boolean') {
throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', ['Invalid option `ignore_last_delimiters`:', 'the value must be a boolean value or an integer,', "got ".concat(JSON.stringify(options.ignore_last_delimiters))], options);
}

if (options.ignore_last_delimiters === true && options.columns === false) {
throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', ['The option `ignore_last_delimiters`', 'requires the activation of the `columns` option'], options);
} // Normalize option `info`


Expand Down Expand Up @@ -1205,7 +1218,13 @@ var Parser = /*#__PURE__*/function (_Transform) {
}, {
key: "__isDelimiter",
value: function __isDelimiter(buf, pos, chr) {
var delimiter = this.options.delimiter;
var _this$options5 = this.options,
delimiter = _this$options5.delimiter,
ignore_last_delimiters = _this$options5.ignore_last_delimiters;

if (ignore_last_delimiters && this.state.record.length === this.options.columns.length - 1) {
return 0;
}

loop1: for (var i = 0; i < delimiter.length; i++) {
var del = delimiter[i];
Expand Down
4 changes: 4 additions & 0 deletions lib/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,10 @@ declare namespace parse {
*/
from_line?: number;
fromLine?: number;
/**
* Don't interpret delimiters as such in the last field according to the number of fields calculated from the number of columns, the option require the presence of the `column` option.
*/
ignore_last_delimiters?: boolean | number;
/**
* Generate two properties `info` and `record` where `info` is a snapshot of the info object at the time the record was created and `record` is the parsed array or object.
*/
Expand Down
23 changes: 22 additions & 1 deletion lib/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,24 @@ class Parser extends Transform {
throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`)
}
}
// Normalize options `ignore_last_delimiters`
if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){
options.ignore_last_delimiters = false
}else if(typeof options.ignore_last_delimiters === 'number'){
options.ignore_last_delimiters = options.ignore_last_delimiters > 0
}else if(typeof options.ignore_last_delimiters !== 'boolean'){
throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [
'Invalid option `ignore_last_delimiters`:',
'the value must be a boolean value or an integer,',
`got ${JSON.stringify(options.ignore_last_delimiters)}`
], options)
}
if(options.ignore_last_delimiters === true && options.columns === false){
throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [
'The option `ignore_last_delimiters`',
'requires the activation of the `columns` option'
], options)
}
// Normalize option `info`
if(options.info === undefined || options.info === null || options.info === false){
options.info = false
Expand Down Expand Up @@ -994,7 +1012,10 @@ class Parser extends Transform {
return numOfCharLeft < requiredLength
}
__isDelimiter(buf, pos, chr){
const {delimiter} = this.options
const {delimiter, ignore_last_delimiters} = this.options
if(ignore_last_delimiters && this.state.record.length === this.options.columns.length - 1){
return 0
}
loop1: for(let i = 0; i < delimiter.length; i++){
const del = delimiter[i]
if(del[0] === chr){
Expand Down
19 changes: 13 additions & 6 deletions test/api.types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,13 @@ describe('API Types', () => {
const keys: string[] = Object.keys(options)
keys.sort().should.eql([
'bom', 'cast', 'cast_date', 'columns', 'columns_duplicates_to_array',
'comment', 'delimiter', 'encoding', 'escape', 'from', 'from_line', 'info', 'ltrim',
'max_record_size', 'objname', 'on_record', 'quote', 'raw',
'record_delimiter', 'relax', 'relax_column_count',
'relax_column_count_less', 'relax_column_count_more', 'rtrim',
'skip_empty_lines', 'skip_lines_with_empty_values',
'skip_lines_with_error', 'to', 'to_line', 'trim'
'comment', 'delimiter', 'encoding', 'escape', 'from', 'from_line',
'ignore_last_delimiters', 'info', 'ltrim', 'max_record_size', 'objname',
'on_record', 'quote', 'raw', 'record_delimiter', 'relax',
'relax_column_count', 'relax_column_count_less',
'relax_column_count_more', 'rtrim', 'skip_empty_lines',
'skip_lines_with_empty_values', 'skip_lines_with_error', 'to',
'to_line', 'trim'
])
})

Expand Down Expand Up @@ -187,6 +188,12 @@ describe('API Types', () => {
options.fromLine = 10
})

it('ignore_last_delimiters', () => {
const options: Options = {}
options.ignore_last_delimiters = true
options.ignore_last_delimiters = 1
})

it('info', () => {
const options: Options = {}
options.info = true
Expand Down
51 changes: 51 additions & 0 deletions test/option.ignore_last_delimiters.coffee
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
parse = require '../lib'

describe 'Option `ignore_last_delimiters`', ->

describe 'validation & normalization', ->

it 'default to `false`', ->
parse().options.ignore_last_delimiters.should.eql false

it 'boolean', ->
parse(ignore_last_delimiters: true, columns: true).options.ignore_last_delimiters.should.eql true
parse(ignore_last_delimiters: false, columns: true).options.ignore_last_delimiters.should.eql false

it 'integer', ->
parse(ignore_last_delimiters: 1, columns: true).options.ignore_last_delimiters.should.eql true
parse(ignore_last_delimiters: 0, columns: true).options.ignore_last_delimiters.should.eql false

it 'throw error with invalid type', ->
(->
parse(ignore_last_delimiters: 'invalid')
).should.throw
code: 'CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS'
message: [
'Invalid option `ignore_last_delimiters`:'
'the value must be a boolean value or an integer,'
'got "invalid"'
].join ' '

it 'requires columns', ->
(->
parse(ignore_last_delimiters: true)
).should.throw
code: 'CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS'
message: [
'The option `ignore_last_delimiters`'
'requires the activation of the `columns` option'
].join ' '

describe 'usage', ->

it 'dont interpret last delimiters', (next) ->
parse '''
a,b,c
1,2,3,4,5
11,22,33,44
''', ignore_last_delimiters: true, columns: true, (err, data) ->
data.should.eql [
{a: '1', b: '2', c: '3,4,5'}
{a: '11', b: '22', c: '33,44'}
] unless err
next err

0 comments on commit 4d70b33

Please sign in to comment.