New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: Add Cockroachdb support without transactions #16394
base: main
Are you sure you want to change the base?
Changes from 108 commits
458bf3a
2fbdddd
a6cee3a
ad7f647
28c7c1a
7589ecd
5719cb3
9686cdb
203e3ca
e05df1f
3187642
8eee4a4
b0ca8d5
5432da2
a0a02e5
d32dda7
5caa5f4
335ddfe
aa92184
848beb7
fb69929
4131a92
5f1c3ce
0969c83
bb1c975
63a72f0
379d476
415c5ea
eaac9d6
4a24e83
d2f6923
b54c111
38019d2
77dcedf
383f60b
df562f1
4cc3361
e20918e
edbc389
9e2f03e
62658e9
23fd4c1
25cce0d
9f8c776
973a465
ec9016a
7dd6975
c561916
3ae9812
9bafaad
fb5fd15
6adc4c7
873c01d
6e60f12
daaffb1
820291c
486912c
6ae87a9
9c813bb
bc5a0ed
44444df
05751c8
7e22c83
5320c56
0af35e6
18cb576
7adf597
2da7559
60434cd
cbec4c2
38ae69a
cec92a7
1db9cc9
d1a5446
dbb5b09
8c94ddc
28a8578
5180503
d5e7895
bc04f94
c4221fd
805ad93
dfddbde
bdf9d55
aa8c069
fe01a8f
e3f13e1
a11833e
9ed6c83
7ab1100
0ff0ad9
d165fa2
d19a8f6
a7fe539
6d71fb6
06599a5
7d0d22e
4349c9e
7e7b109
53d776f
b3179a0
678f110
3bdbaf7
fd3692e
e663bc0
1ec03e4
e2beef3
fcd8b9b
abf412b
7db6d8b
c26e6ab
39dd7a7
adee21f
23d33e8
1f78847
f669df1
e022e64
2b1003f
f7a8507
5226fe8
6eb1b18
5255738
66dea65
ef9a12b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
services: | ||
cockroachdb-latest: | ||
container_name: sequelize-cockroachdb-latest | ||
image: cockroachdb/cockroach:v23.1.6 | ||
ports: | ||
- 26257:26257 | ||
- 8080:8080 | ||
command: start-single-node --cluster-name=cockroach-sequelize-example --advertise-addr='localhost' --insecure | ||
environment: | ||
COCKROACH_DATABASE: public | ||
volumes: | ||
- cockroach-single:/cockroach/cockroach-data | ||
- ~/init-scripts:/docker-entrypoint-initdb.d | ||
healthcheck: | ||
test: curl --fail http://localhost:8080 || exit 1 | ||
interval: 3s | ||
timeout: 1s | ||
retries: 10 | ||
networks: | ||
default: | ||
name: sequelize-cockroachdb-latest-network | ||
volumes: | ||
cockroach-single: |
Original file line number | Diff line number | Diff line change | ||||||||
---|---|---|---|---|---|---|---|---|---|---|
@@ -0,0 +1,25 @@ | ||||||||||
#!/usr/bin/env bash | ||||||||||
set -Eeuxo pipefail # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ | ||||||||||
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" # https://stackoverflow.com/a/17744637 | ||||||||||
|
||||||||||
docker compose -p sequelize-cockroachdb-latest down --remove-orphans | ||||||||||
docker compose -p sequelize-cockroachdb-latest up -d | ||||||||||
|
||||||||||
./../../wait-until-healthy.sh sequelize-cockroachdb-latest | ||||||||||
|
||||||||||
# Optimising the single node cluster | ||||||||||
# Read more here https://cockroachlabs.com/docs/stable/local-testing.html#use-a-local-single-node-cluster-with-in-memory-storage | ||||||||||
docker exec sequelize-cockroachdb-latest ./cockroach sql --insecure --execute=" | ||||||||||
SET CLUSTER SETTING kv.raft_log.disable_synchronization_unsafe = true; | ||||||||||
SET CLUSTER SETTING kv.range_merge.queue_interval = '50ms'; | ||||||||||
SET CLUSTER SETTING jobs.registry.interval.gc = '30s'; | ||||||||||
SET CLUSTER SETTING jobs.registry.interval.cancel = '180s'; | ||||||||||
SET CLUSTER SETTING jobs.retention_time = '15s'; | ||||||||||
SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false; | ||||||||||
SET CLUSTER SETTING kv.range_split.by_load_merge_delay = '5s'; | ||||||||||
ALTER RANGE default CONFIGURE ZONE USING "gc.ttlseconds" = 600; | ||||||||||
ALTER DATABASE system CONFIGURE ZONE USING "gc.ttlseconds" = 600; | ||||||||||
Comment on lines
+20
to
+21
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||
DROP DATABASE IF EXISTS sequelize_test; | ||||||||||
CREATE DATABASE IF NOT EXISTS sequelize_test; | ||||||||||
CREATE USER IF NOT EXISTS sequelize_test; | ||||||||||
GRANT admin TO sequelize_test;" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
#!/usr/bin/env bash | ||
set -Eeuxo pipefail # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ | ||
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" # https://stackoverflow.com/a/17744637 | ||
|
||
docker compose -p sequelize-cockroachdb-latest down --remove-orphans | ||
|
||
echo "Cockroach db cluster stopped" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
services: | ||
cockroachdb-oldest: | ||
container_name: sequelize-cockroachdb-oldest | ||
image: cockroachdb/cockroach:v22.2.0 | ||
ports: | ||
- 26257:26257 | ||
- 8080:8080 | ||
command: start-single-node --cluster-name=cockroach-sequelize-example --advertise-addr='localhost' --insecure | ||
environment: | ||
COCKROACH_DATABASE: public | ||
volumes: | ||
- cockroach-single:/cockroach/cockroach-data-oldest | ||
- ~/init-scripts:/docker-entrypoint-initdb.d | ||
healthcheck: | ||
test: curl --fail http://localhost:8080 || exit 1 | ||
interval: 3s | ||
timeout: 1s | ||
retries: 10 | ||
networks: | ||
default: | ||
name: sequelize-cockroachdb-oldest-network | ||
volumes: | ||
cockroach-single: |
Original file line number | Diff line number | Diff line change | ||||||||
---|---|---|---|---|---|---|---|---|---|---|
@@ -0,0 +1,25 @@ | ||||||||||
#!/usr/bin/env bash | ||||||||||
set -Eeuxo pipefail # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ | ||||||||||
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" # https://stackoverflow.com/a/17744637 | ||||||||||
|
||||||||||
docker compose -p sequelize-cockroachdb-oldest down --remove-orphans | ||||||||||
docker compose -p sequelize-cockroachdb-oldest up -d | ||||||||||
|
||||||||||
./../../wait-until-healthy.sh sequelize-cockroachdb-oldest | ||||||||||
|
||||||||||
# Optimising the single node cluster | ||||||||||
# Read more here https://cockroachlabs.com/docs/stable/local-testing.html#use-a-local-single-node-cluster-with-in-memory-storage | ||||||||||
docker exec sequelize-cockroachdb-oldest ./cockroach sql --insecure --execute=" | ||||||||||
SET CLUSTER SETTING kv.raft_log.disable_synchronization_unsafe = true; | ||||||||||
SET CLUSTER SETTING kv.range_merge.queue_interval = '50ms'; | ||||||||||
SET CLUSTER SETTING jobs.registry.interval.gc = '30s'; | ||||||||||
SET CLUSTER SETTING jobs.registry.interval.cancel = '180s'; | ||||||||||
SET CLUSTER SETTING jobs.retention_time = '15s'; | ||||||||||
SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false; | ||||||||||
SET CLUSTER SETTING kv.range_split.by_load_merge_delay = '5s'; | ||||||||||
ALTER RANGE default CONFIGURE ZONE USING "gc.ttlseconds" = 600; | ||||||||||
ALTER DATABASE system CONFIGURE ZONE USING "gc.ttlseconds" = 600; | ||||||||||
Comment on lines
+20
to
+21
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||
DROP DATABASE IF EXISTS sequelize_test; | ||||||||||
CREATE DATABASE IF NOT EXISTS sequelize_test; | ||||||||||
CREATE USER IF NOT EXISTS sequelize_test; | ||||||||||
GRANT admin TO sequelize_test;" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
#!/usr/bin/env bash | ||
set -Eeuxo pipefail # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ | ||
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" # https://stackoverflow.com/a/17744637 | ||
|
||
docker compose -p sequelize-cockroachdb-oldest down --remove-orphans | ||
|
||
echo "Cockroach db cluster stopped" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
import { PostgresConnectionManager } from '../postgres/connection-manager'; | ||
|
||
export class CockroachdbConnectionManager extends PostgresConnectionManager {} | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'll need to review this file later |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
import assert from 'node:assert'; | ||
import wkx from 'wkx'; | ||
import * as BaseTypes from '../abstract/data-types'; | ||
import { getDataTypeParser } from '../abstract/data-types-utils'; | ||
import { buildRangeParser } from '../postgres/range'; | ||
import type { CockroachDbDialect } from './index'; | ||
|
||
export function registerCockroachDbDataTypeParsers(dialect: CockroachDbDialect) { | ||
// dateonly | ||
dialect.registerDataTypeParser(['date'], (value: unknown) => { | ||
return value; | ||
}); | ||
|
||
dialect.registerDataTypeParser(['timestamptz', 'timestamp'], (value: unknown) => { | ||
// override default parser to prevent returning a Date object (which is the default behavior in pg). | ||
// return dates as string, not Date objects. Different implementations could be used instead (such as Temporal, dayjs) | ||
return value; | ||
}); | ||
|
||
dialect.registerDataTypeParser(['decimal', 'numeric'], (value: string) => { | ||
if (value === null) { | ||
return null; | ||
} | ||
|
||
if (value === 'NaN') { | ||
return Number.NaN; | ||
} | ||
|
||
return value; | ||
}); | ||
|
||
/** | ||
* Cockroachdb's ALIAS for INTEGER and BIGINT defaults to int(64-bit value). So while retreiving the values from the database both values are | ||
* returned as int8. While parsing them we cannot be sure if the expected behavior is of INTEGER or BIGINT so we make sure if the number is | ||
* < than Number.MAX_SAFE_INTEGER --> Parse it as a number | ||
* > than Number.MAX_SAFE_INTEGER --> Parse it as a string | ||
* Note: Storing INTEGERS as INT4 and BIGINT as int8 would result in an anti-pattern since Cockroachdb uses unique_row_id() to generate a unique ID for | ||
* each row by default to ensure global uniqueness, reseliency and consistency. | ||
*/ | ||
dialect.registerDataTypeParser(['int8'], (value: string) => { | ||
if (value === null) { | ||
return null; | ||
} | ||
|
||
const parsedValue = BigInt(value); | ||
|
||
if (parsedValue > Number.MAX_SAFE_INTEGER || parsedValue < Number.MIN_SAFE_INTEGER) { | ||
return String(value); | ||
} else if (Number(value) >= Number.MIN_SAFE_INTEGER) { | ||
return Number(value); | ||
} | ||
|
||
return String(value); | ||
|
||
}); | ||
|
||
dialect.registerDataTypeParser(['geography'], (value: unknown) => { | ||
assert(typeof value === 'string', 'Expected geography value to be a string'); | ||
|
||
const b = Buffer.from(value, 'hex'); | ||
|
||
return wkx.Geometry.parse(b).toGeoJSON({ shortCrs: true }); | ||
}); | ||
|
||
dialect.registerDataTypeParser(['geometry'], (value: unknown) => { | ||
assert(typeof value === 'string', 'Expected geometry value to be a string'); | ||
|
||
const b = Buffer.from(value, 'hex'); | ||
|
||
return wkx.Geometry.parse(b).toGeoJSON({ shortCrs: true }); | ||
}); | ||
|
||
const parseInteger = getDataTypeParser(dialect, BaseTypes.INTEGER); | ||
dialect.registerDataTypeParser(['int4range'], buildRangeParser(parseInteger)); | ||
|
||
const parseBigInt = getDataTypeParser(dialect, BaseTypes.BIGINT); | ||
dialect.registerDataTypeParser(['int8range'], buildRangeParser(parseBigInt)); | ||
|
||
const parseDecimal = getDataTypeParser(dialect, BaseTypes.DECIMAL); | ||
dialect.registerDataTypeParser(['numrange'], buildRangeParser(parseDecimal)); | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This should be added to the scopes in .github/workflows/pr-title.yml as well