forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 0
/
build.gradle
129 lines (108 loc) · 4.72 KB
/
build.gradle
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
import java.nio.file.Paths
plugins {
id 'airbyte-docker'
id 'airbyte-python'
}
airbytePython {
moduleDirectory 'normalization'
}
dependencies {
implementation project(':airbyte-workers')
}
// we need to access the sshtunneling script from airbyte-workers for ssh support
task copySshScript(type: Copy, dependsOn: [project(':airbyte-workers').processResources]) {
from "${project(':airbyte-workers').buildDir}/resources/main"
into "${buildDir}"
include "sshtunneling.sh"
}
// make sure the copy task above worked (if it fails, it fails silently annoyingly)
task checkSshScriptCopy(type: Task, dependsOn: copySshScript) {
doFirst {
assert file("${buildDir}/sshtunneling.sh").exists() :
"Copy of sshtunneling.sh failed, check that it is present in airbyte-workers."
}
}
airbyteDocker.dependsOn(checkSshScriptCopy)
assemble.dependsOn(checkSshScriptCopy)
test.dependsOn(checkSshScriptCopy)
integrationTest.dependsOn(build)
static def getDockerfile(String customConnector) {
return "${customConnector}.Dockerfile"
}
static def getDockerImageName(String customConnector) {
return "airbyte/normalization-${customConnector}"
}
static def getImageNameWithTag(String customConnector) {
return "${getDockerImageName(customConnector)}:dev"
}
def buildAirbyteDocker(String customConnector) {
def baseCommand = ['docker', 'build', '.', '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector)]
return {
commandLine baseCommand
}
}
task airbyteDockerMSSql(type: Exec, dependsOn: checkSshScriptCopy) {
configure buildAirbyteDocker('mssql')
dependsOn assemble
}
task airbyteDockerMySql(type: Exec, dependsOn: checkSshScriptCopy) {
configure buildAirbyteDocker('mysql')
dependsOn assemble
}
task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) {
configure buildAirbyteDocker('oracle')
dependsOn assemble
}
task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) {
configure buildAirbyteDocker('clickhouse')
dependsOn assemble
}
task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) {
configure buildAirbyteDocker('snowflake')
dependsOn assemble
}
task airbyteDockerDatabricks(type: Exec, dependsOn: checkSshScriptCopy) {
configure buildAirbyteDocker('databricks')
dependsOn assemble
}
task airbyteDockerRedshift(type: Exec, dependsOn: checkSshScriptCopy) {
configure buildAirbyteDocker('redshift')
dependsOn assemble
}
airbyteDocker.dependsOn(airbyteDockerMSSql)
airbyteDocker.dependsOn(airbyteDockerMySql)
airbyteDocker.dependsOn(airbyteDockerOracle)
airbyteDocker.dependsOn(airbyteDockerClickhouse)
airbyteDocker.dependsOn(airbyteDockerSnowflake)
airbyteDocker.dependsOn(airbyteDockerDatabricks)
airbyteDocker.dependsOn(airbyteDockerRedshift)
task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) {
module = "pytest"
command = "-s integration_tests"
dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker'
dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker'
dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker'
dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker'
dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker'
dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker'
dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker'
dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker'
dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker'
dependsOn ':airbyte-integrations:connectors:destination-databricks:airbyteDocker'
}
// DATs have some additional tests that exercise normalization code paths,
// so we want to run these in addition to the base-normalization integration tests.
// If you add more items here, make sure to also to have CI fetch their credentials.
// See git history for an example.
integrationTest.dependsOn(":airbyte-integrations:connectors:destination-bigquery:integrationTest")
integrationTest.dependsOn(":airbyte-integrations:connectors:destination-postgres:integrationTest")
integrationTest.dependsOn(":airbyte-integrations:connectors:destination-snowflake:integrationTest")
integrationTest.dependsOn("customIntegrationTestPython")
customIntegrationTests.dependsOn("customIntegrationTestPython")
// TODO fix and use https://github.com/airbytehq/airbyte/issues/3192 instead
task('mypyCheck', type: PythonTask) {
module = "mypy"
command = "normalization --config-file ${project.rootProject.file('pyproject.toml').absolutePath}"
dependsOn 'blackFormat'
}
check.dependsOn mypyCheck