Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

types: Add Expression Typings to Pipeline Stages #11370

Merged
merged 32 commits into from
Jun 17, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
2582959
initial
Uzlopak Feb 9, 2022
4b1e8e8
fix linting issues
Uzlopak Feb 9, 2022
d330145
fix linter error
Uzlopak Feb 9, 2022
f0ab5c8
Arithmetic Expression Operators
Uzlopak Feb 9, 2022
d0f466f
Text Expression Operators
Uzlopak Feb 9, 2022
fdada6b
+ Trigonometry Expression Operators
Uzlopak Feb 9, 2022
0291e9b
add comment
Uzlopak Feb 9, 2022
c667b50
add version info
Uzlopak Feb 9, 2022
519c314
only add and subtract can handle dates
Uzlopak Feb 9, 2022
742c951
add DateExpressionOperators
Uzlopak Feb 10, 2022
e2d1e6a
add boolean and comparison
Uzlopak Feb 10, 2022
29a8837
add ConditionalExpressionOperator
Uzlopak Feb 10, 2022
a8f26b4
add ArrayExpression
Uzlopak Feb 10, 2022
a17370e
some minor changes
Uzlopak Feb 10, 2022
ebcad04
add StringExpressionOperators
Uzlopak Feb 10, 2022
40acd28
Merge branch 'master' into types-expression
Uzlopak Feb 16, 2022
06f879e
Merge branch 'main' into types-expression
Uzlopak May 30, 2022
15dcd8e
remove duplicate Expression.test.ts
Uzlopak May 30, 2022
8f3f784
add more
Uzlopak May 30, 2022
be4a808
add Set Expression Operators
Uzlopak May 30, 2022
7cef3c5
add Window Expressions
Uzlopak May 30, 2022
e8e6842
add type expressions
Uzlopak May 31, 2022
08270f4
add DataSizeOperator
Uzlopak May 31, 2022
1ef01c0
add last operation set
Uzlopak May 31, 2022
238bb90
add some tests
Uzlopak May 31, 2022
f57e007
add tests, fix typings
Uzlopak May 31, 2022
510412f
add more tests, fix more
Uzlopak May 31, 2022
4635fcd
fix more
Uzlopak May 31, 2022
c94d2bb
fix linting
Uzlopak May 31, 2022
16deac9
auto format .eslintrc.json
Uzlopak Jun 2, 2022
365f39f
fix .eslint.json and linting errors
Uzlopak Jun 2, 2022
e3dca30
Merge branch 'master' into types-expression
Uzlopak Jun 2, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
95 changes: 61 additions & 34 deletions .eslintrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,43 +4,70 @@
],
"ignorePatterns": [
"docs",
"tools",
"dist",
"test/files/*"
"website.js",
"test/files/*",
"benchmarks"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we really want to avoid linting benchmarks and website.js? Doesn't seem like the right thing to do, but I might be missing something

],
"overrides": [{
"files": [
"**/*.{ts,tsx}"
],
"extends": [
"plugin:@typescript-eslint/eslint-recommended",
"plugin:@typescript-eslint/recommended"
],
"plugins": [
"@typescript-eslint"
],
"rules": {
"@typescript-eslint/triple-slash-reference": "off",
"spaced-comment": ["error", "always", {
"markers": ["/"]
}],
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/ban-types": "off",
"@typescript-eslint/no-unused-vars": "off",
"@typescript-eslint/explicit-module-boundary-types": "off",
"@typescript-eslint/indent": ["error", 2, {
"SwitchCase": 1
}],
"@typescript-eslint/prefer-optional-chain": "error",
"@typescript-eslint/brace-style": "error",
"@typescript-eslint/no-dupe-class-members": "error",
"@typescript-eslint/no-redeclare": "error",
"@typescript-eslint/type-annotation-spacing": "error",
"@typescript-eslint/object-curly-spacing": ["error", "always"],
"@typescript-eslint/semi": "error",
"@typescript-eslint/space-before-function-paren": ["error", "never"],
"@typescript-eslint/space-infix-ops": "off"
"overrides": [
{
"files": [
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are these changes necessary?

"**/*.{ts,tsx}"
],
"extends": [
"plugin:@typescript-eslint/eslint-recommended",
"plugin:@typescript-eslint/recommended"
],
"plugins": [
"@typescript-eslint"
],
"rules": {
"@typescript-eslint/triple-slash-reference": "off",
"spaced-comment": [
"error",
"always",
{
"block": {
"markers": [
"!"
],
"balanced": true
},
"markers": [
"/"
]
}
],
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/ban-types": "off",
"@typescript-eslint/no-unused-vars": "off",
"@typescript-eslint/explicit-module-boundary-types": "off",
"@typescript-eslint/indent": [
"error",
2,
{
"SwitchCase": 1
}
],
"@typescript-eslint/prefer-optional-chain": "error",
"@typescript-eslint/brace-style": "error",
"@typescript-eslint/no-dupe-class-members": "error",
"@typescript-eslint/no-redeclare": "error",
"@typescript-eslint/type-annotation-spacing": "error",
"@typescript-eslint/object-curly-spacing": [
"error",
"always"
],
"@typescript-eslint/semi": "error",
"@typescript-eslint/space-before-function-paren": [
"error",
"never"
],
"@typescript-eslint/space-infix-ops": "off"
}
}
}],
],
"plugins": [
"mocha-no-only"
],
Expand Down
243 changes: 242 additions & 1 deletion test/types/PipelineStage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -128,4 +128,245 @@ const unwind1: PipelineStage = { $unwind: '$sizes' };
const unwind2: PipelineStage = { $unwind: { path: '$sizes' } };
const unwind3: PipelineStage = { $unwind: { path: '$sizes', includeArrayIndex: 'arrayIndex' } };
const unwind4: PipelineStage = { $unwind: { path: '$sizes', preserveNullAndEmptyArrays: true } };
const unwind5: PipelineStage = { $unwind: { path: '$sizes', preserveNullAndEmptyArrays: true } };
const unwind5: PipelineStage = { $unwind: { path: '$sizes', preserveNullAndEmptyArrays: true } };

const redact1: PipelineStage = {
$redact: {
$cond: {
if: { $gt: [{ $size: { $setIntersection: ['$tags', 'userAccess'] } }, 0] },
then: '$$DESCEND',
else: '$$PRUNE'
}
}
};

const redact2: PipelineStage = {
$redact: {
$cond: {
if: { $eq: ['$level', 5] },
then: '$$PRUNE',
else: '$$DESCEND'
}
}
};
const replaceRoot: PipelineStage = { $replaceRoot: { newRoot: { $mergeObjects: [{ dogs: 0, cats: 0, birds: 0, fish: 0 }, '$pets'] } } };

const project1: PipelineStage = { $project: { contact: 1, 'contact.address.country': 1 } };
const project2: PipelineStage = { $project: { 'contact.address.country': 1, contact: 1 } };
const project3: PipelineStage = { $project: { author: { first: 0 }, lastModified: 0 } };
const project4: PipelineStage = {
$project: {
title: 1,
'author.first': 1,
'author.last': 1,
'author.middle': {
$cond: {
if: { $eq: ['', '$author.middle'] },
then: '$$REMOVE',
else: '$author.middle'
}
}
}
};
const project5: PipelineStage = { $project: { 'stop.title': 1 } };
const project6: PipelineStage = { $project: { stop: { title: 1 } } };
const project7: PipelineStage = {
$project: {
title: 1,
isbn: {
prefix: { $substr: ['$isbn', 0, 3] },
group: { $substr: ['$isbn', 3, 2] },
publisher: { $substr: ['$isbn', 5, 4] },
title: { $substr: ['$isbn', 9, 3] },
checkDigit: { $substr: ['$isbn', 12, 1] }
},
lastName: '$author.last',
copiesSold: '$copies'
}
};
const project8: PipelineStage = { $project: { myArray: ['$x', '$y'] } };
const project9: PipelineStage = { $project: { x: '$name.0', _id: 0 } };
const project10: PipelineStage = { $project: { stdDev: { $stdDevPop: '$scores.score' } } };
const project11: PipelineStage = {
$project:
{
item: 1,
comparisonResult: { $strcasecmp: ['$quarter', '13q4'] }
}
};
const project12: PipelineStage = {
$project: {
name: 1,
length: { $strLenBytes: '$name' }
}
};
const project13: PipelineStage = {
$project: {
name: 1,
length: { $strLenCP: '$name' }
}
};

const project14: PipelineStage = {
$project:
{
item: 1,
yearSubstring: { $substr: ['$quarter', 0, 2] },
quarterSubtring: { $substr: ['$quarter', 2, -1] }
}
};
const project15: PipelineStage = { $project: { item: 1, result: { $not: [{ $gt: ['$qty', 250] }] } } };

const sort1: PipelineStage = { $sort: { count: -1 } };
const sortByCount1: PipelineStage = { $sortByCount: '$tags' };
const sortByCount2: PipelineStage = { $sortByCount: { $mergeObjects: ['$employee', '$business'] } };

const set1: PipelineStage = { $set: { 'specs.fuel_type': 'unleaded' } };
const set2: PipelineStage = { $set: { cats: 20 } };
const set3: PipelineStage = { $set: { _id: '$item', item: 'fruit' } };
const set4: PipelineStage = { $set: { homework: { $concatArrays: ['$homework', [7]] } } };

const merge1: PipelineStage = { $merge: { into: 'newDailySales201905', on: 'salesDate' } };
const merge2: PipelineStage = { $merge: { into: 'newrestaurants', on: ['date', 'postcode'], whenMatched: 'replace', whenNotMatched: 'insert' } };
const merge3: PipelineStage = { $merge: { into: { db: 'reporting', coll: 'budgets' }, on: '_id', whenMatched: 'replace', whenNotMatched: 'insert' } };
const merge4: PipelineStage = { $merge: { into: { db: 'reporting', coll: 'orgArchive' }, on: ['dept', 'fiscal_year'], whenMatched: 'fail' } };
const merge5: PipelineStage = { $merge: { into: 'quarterlyreport', on: '_id', whenMatched: 'merge', whenNotMatched: 'insert' } };
const merge6: PipelineStage = {
$merge: {
into: 'monthlytotals',
on: '_id',
whenMatched: [
{
$addFields: {
thumbsup: { $add: ['$thumbsup', '$$new.thumbsup'] },
thumbsdown: { $add: ['$thumbsdown', '$$new.thumbsdown'] }
}
}],
whenNotMatched: 'insert'
}
};

const match1: PipelineStage = { $match: { $or: [{ score: { $gt: 70, $lt: 90 } }, { views: { $gte: 1000 } }] } };
const match2: PipelineStage = { $match: { test: 'bla' } };
const match3: PipelineStage = { $match: { test: { $or: [{ score: { $gt: 70, $lt: 90 } }, { views: { $gte: 1000 } }] } } };
const match4: PipelineStage = { $match: { $and: [{ score: { $gt: 70, $lt: 90 } }, { views: { $gte: 1000 } }] } };
const match5: PipelineStage = { $match: { test: { $and: [{ score: { $gt: 70, $lt: 90 } }, { views: { $gte: 1000 } }] } } };
const addFields7: PipelineStage = { $addFields: { convertedQty: { $toLong: '$qty' } } };

const setWindowFields1: PipelineStage = {
$setWindowFields: {
partitionBy: '$state',
sortBy: { orderDate: 1 },
output: {
stdDevPopQuantityForState: {
$stdDevPop: '$quantity',
window: {
documents: ['unbounded', 'current']
}
}
}
}
};

const setWindowFields2: PipelineStage = {
$setWindowFields: {
partitionBy: '$state',
sortBy: { orderDate: 1 },
output: {
stdDevSampQuantityForState: {
$stdDevSamp: '$quantity',
window: {
documents: ['unbounded', 'current']
}
}
}
}
};

const setWindowFields3: PipelineStage = {
$setWindowFields: {
partitionBy: '$stock',
sortBy: { date: 1 },
output: {
expMovingAvgForStock: {
$expMovingAvg: { input: '$price', N: 2 }
}
}
}
};

const setWindowFields4: PipelineStage = {
$setWindowFields: {
partitionBy: '$stock',
sortBy: { date: 1 },
output: {
expMovingAvgForStock: {
$expMovingAvg: { input: '$price', alpha: 0.75 }
}
}
}
};

const group1: PipelineStage = { $group: { _id: null, ageStdDev: { $stdDevSamp: '$age' } } };
const group2: PipelineStage = {
$group: {
_id: { x: '$x' },
y: { $first: '$y' }
}
};
const group3: PipelineStage = {
$group: {
_id: null,
count: { $count: {} }
}
};
const group4: PipelineStage = {
$group:
{
_id: '$item',
totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } }
}
};
const group5: PipelineStage = {
$group: {
_id: null,
totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } },
averageQuantity: { $avg: '$quantity' },
count: { $sum: 1 }
}
};
const group6: PipelineStage = { $group: { _id: '$author', books: { $push: '$title' } } };

const stages1: PipelineStage[] = [
// First Stage
{
$match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } }
},
// Second Stage
{
$group: {
_id: { $dateToString: { format: '%Y-%m-%d', date: '$date' } },
totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } },
averageQuantity: { $avg: '$quantity' },
count: { $sum: 1 }
}
},
// Third Stage
{
$sort: { totalSaleAmount: -1 }
}
];

const stages2: PipelineStage[] = [
// First Stage
{
$group: { _id: '$author', books: { $push: '$$ROOT' } }
},
// Second Stage
{
$addFields:
{
totalCopies: { $sum: '$books.copies' }
}
}
];