diff --git a/arenadata/Dockerfile b/arenadata/Dockerfile new file mode 100644 index 0000000000..f64438d47e --- /dev/null +++ b/arenadata/Dockerfile @@ -0,0 +1,34 @@ +FROM hub.adsw.io/library/gpdb6_regress:latest as base + +# install go, ginkgo and keep env variables which may be used as a part of base image +RUN yum install -y go +ENV GOPATH=$HOME/go +ENV PATH=$PATH:/usr/local/go/bin:$GOPATH/bin +RUN go install github.com/onsi/ginkgo/ginkgo@latest + +# leave pxf artifacts dir env also +ENV OUTPUT_ARTIFACT_DIR="pxf_tarball" + +# remove unnecessary artifacts and create symlinks +# concource scripts expects gpdb and pxf placed in the same folder +RUN rm /home/gpadmin/bin_gpdb/server-*.tar.gz && \ + mkdir /tmp/build && \ + ln -s /home/gpadmin/gpdb_src /tmp/build/gpdb_src && \ + ln -s /home/gpadmin/bin_gpdb /tmp/build/bin_gpdb +# default working dir - the place where all sources and artifacts are placed +WORKDIR /tmp/build + +# create separate image with files we don't want to keep in base image +FROM base as build +COPY . /tmp/build/pxf_src +RUN source gpdb_src/concourse/scripts/common.bash && \ + install_gpdb && \ + source '/usr/local/greenplum-db-devel/greenplum_path.sh' && \ + mkdir ${OUTPUT_ARTIFACT_DIR} && \ + export SKIP_FDW_BUILD_REASON=0 && \ + pxf_src/concourse/scripts/compile_pxf.bash + +# create test image which prepares base image and keeps only pxf artifacts from build image +FROM base as test +COPY --from=build /tmp/build/${OUTPUT_ARTIFACT_DIR}/pxf.tar.gz /tmp/build/${OUTPUT_ARTIFACT_DIR}/ +COPY --from=build /tmp/build/pxf_src /tmp/build/pxf_src diff --git a/arenadata/README.md b/arenadata/README.md new file mode 100644 index 0000000000..d43ae53a8f --- /dev/null +++ b/arenadata/README.md @@ -0,0 +1,15 @@ +## How to build PXF Docker image +From the root pxf folder run: +```bash +docker build -t gpdb6_pxf_regress:latest -f arenadata/Dockerfile . +``` +This will build an image called `gpdb6_pxf_regress` with the tag `latest`. This image is based on `gpdb6_regress:latest`, which additionally contains pxf sources and pxf artifacts tarball in `/tmp/build/pxf_src` and `/tmp/build/pxf_tarball` folders respectively. + +## How to test PXF +During the image building phase `compile_pxf.bash` script additionally calls `test` make target, which calls `make -C cli/go/src/pxf-cli test` and `make -C server test` commands. +To additionally test `fdw` and `external-table` parts you may call: +```bash +docker run --rm -it \ + --privileged --sysctl kernel.sem="500 1024000 200 4096" \ + gpdb6_pxf_regress:latest /tmp/build/pxf_src/arenadata/test_in_docker.sh +``` diff --git a/arenadata/test_in_docker.sh b/arenadata/test_in_docker.sh new file mode 100755 index 0000000000..dc97e3eb3a --- /dev/null +++ b/arenadata/test_in_docker.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# This script depends on hub.adsw.io/library/gpdb6_pxf_regress +set -exo pipefail + +# manually prepare gpadmin user; test_pxf.bash doesn't tweak gpadmin folder permissions and ssh keys +./gpdb_src/concourse/scripts/setup_gpadmin_user.bash +# unpack gpdb and pxf; run gpdb cluster and pxf server +/tmp/build/pxf_src/concourse/scripts/test_pxf.bash +# tweak necessary folders to run regression tests later +chown gpadmin:gpadmin -R /usr/local/greenplum-db-devel +chown gpadmin:gpadmin -R /tmp/build/pxf_src + +# test fdw and external-table +su - gpadmin -c " + source '/usr/local/greenplum-db-devel/greenplum_path.sh'; + source '/home/gpadmin/gpdb_src/gpAux/gpdemo/gpdemo-env.sh'; + cd /tmp/build/pxf_src/fdw && + make install && + make installcheck && + cd ../external-table/ && + make install && + make installcheck; +" diff --git a/external-table/src/libchurl.c b/external-table/src/libchurl.c index 40e034805f..e616781487 100644 --- a/external-table/src/libchurl.c +++ b/external-table/src/libchurl.c @@ -473,7 +473,6 @@ churl_read_check_connectivity(CHURL_HANDLE handle) Assert(!context->upload); fill_internal_buffer(context, 1); - check_response(context); } /* @@ -626,6 +625,8 @@ multi_perform(churl_context *context) if (curl_error != CURLM_OK) elog(ERROR, "internal error: curl_multi_perform failed (%d - %s)", curl_error, curl_easy_strerror(curl_error)); + + check_response(context); } static bool @@ -653,8 +654,6 @@ flush_internal_buffer(churl_context *context) multi_perform(context); } - check_response(context); - if ((context->curl_still_running == 0) && ((context_buffer->top - context_buffer->bot) > 0)) elog(ERROR, "failed sending to remote component %s", get_dest_address(context->curl_handle)); @@ -709,8 +708,6 @@ finish_upload(churl_context *context) */ while (context->curl_still_running != 0) multi_perform(context); - - check_response(context); } static void diff --git a/external-table/src/pxfbridge.c b/external-table/src/pxfbridge.c index 713c1d306a..26ab7274e9 100644 --- a/external-table/src/pxfbridge.c +++ b/external-table/src/pxfbridge.c @@ -40,7 +40,7 @@ gpbridge_cleanup(gphadoop_context *context) if (context == NULL) return; - churl_cleanup(context->churl_handle, false); + churl_cleanup(context->churl_handle, context->after_error); context->churl_handle = NULL; churl_headers_cleanup(context->churl_headers); diff --git a/external-table/src/pxfbridge.h b/external-table/src/pxfbridge.h index 96517ae913..63b8224131 100644 --- a/external-table/src/pxfbridge.h +++ b/external-table/src/pxfbridge.h @@ -43,6 +43,7 @@ typedef struct ProjectionInfo *proj_info; List *quals; bool completed; + bool after_error; } gphadoop_context; /* diff --git a/external-table/src/pxfprotocol.c b/external-table/src/pxfprotocol.c index 4ccb1b96bb..2f7d3472bd 100644 --- a/external-table/src/pxfprotocol.c +++ b/external-table/src/pxfprotocol.c @@ -26,6 +26,7 @@ #include "access/fileam.h" #endif #include "utils/elog.h" +#include "utils/resowner.h" /* define magic module unless run as a part of test cases */ #ifndef UNIT_TESTING @@ -154,6 +155,21 @@ pxfprotocol_import(PG_FUNCTION_ARGS) PG_RETURN_INT32(bytes_read); } +static void +url_curl_abort_callback(ResourceReleasePhase phase, + bool isCommit, + bool isTopLevel, + void *arg) +{ + gphadoop_context *context = arg; + + if (phase != RESOURCE_RELEASE_AFTER_LOCKS || isCommit || !isTopLevel) + return; + + context->after_error = true; + cleanup_context(context); +} + /* * Allocates context and sets values for the segment */ @@ -190,6 +206,8 @@ create_context(PG_FUNCTION_ARGS, bool is_import) context->proj_info = proj_info; context->quals = filter_quals; context->completed = false; + context->after_error = false; + RegisterResourceReleaseCallback(url_curl_abort_callback, context); return context; } @@ -201,6 +219,7 @@ cleanup_context(gphadoop_context *context) { if (context != NULL) { + UnregisterResourceReleaseCallback(url_curl_abort_callback, context); gpbridge_cleanup(context); pfree(context->uri.data); pfree(context); diff --git a/fdw/libchurl.c b/fdw/libchurl.c index 6b2fa93a49..083ad24bc2 100644 --- a/fdw/libchurl.c +++ b/fdw/libchurl.c @@ -480,7 +480,6 @@ churl_read_check_connectivity(CHURL_HANDLE handle) Assert(!context->upload); fill_internal_buffer(context, 1); - check_response(context); } /* @@ -633,6 +632,8 @@ multi_perform(churl_context *context) if (curl_error != CURLM_OK) elog(ERROR, "internal error: curl_multi_perform failed (%d - %s)", curl_error, curl_easy_strerror(curl_error)); + + check_response(context); } static bool @@ -717,8 +718,6 @@ finish_upload(churl_context *context) */ while (context->curl_still_running != 0) multi_perform(context); - - check_response(context); } static void diff --git a/server/build.gradle b/server/build.gradle index b110e59a81..fb3434d61b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -20,6 +20,18 @@ buildscript { repositories { mavenCentral() + maven { + url "https://rt.adsw.io/artifactory/maven-arenadata-release/" + mavenContent { + releasesOnly() + } + } + maven { + url "https://rt.adsw.io/artifactory/maven-arenadata-snapshot/" + mavenContent { + snapshotsOnly() + } + } } } @@ -40,6 +52,18 @@ allprojects { repositories { mavenCentral() + maven { + url "https://rt.adsw.io/artifactory/maven-arenadata-release/" + mavenContent { + releasesOnly() + } + } + maven { + url "https://rt.adsw.io/artifactory/maven-arenadata-snapshot/" + mavenContent { + snapshotsOnly() + } + } } } @@ -95,6 +119,18 @@ configure(javaProjects) { dependency("org.apache.htrace:htrace-core:3.1.0-incubating") dependency("org.apache.htrace:htrace-core4:4.0.1-incubating") + // --- bump log4j2 to 2.17.1 for CVE-2021-44228, CVE-2021-45046, and CVE-2021-45105 fixes, + // more details: https://logging.apache.org/log4j/2.x/security.html + // revert once org.springframework.boot:spring-boot-starter-log4j2 is upgraded to bundle log4j2:2.17.1+ + dependencySet(group:"org.apache.logging.log4j", version:"2.17.1") { + entry("log4j-jul") + entry("log4j-api") + entry("log4j-core") + entry("log4j-spring-boot") + } + dependency("org.apache.logging.log4j:log4j-slf4j-impl:2.17.1") + // --- end of CVE patch + dependency("org.apache.zookeeper:zookeeper:3.4.6") dependency("org.codehaus.woodstox:stax2-api:3.1.4") dependency("org.datanucleus:datanucleus-api-jdo:4.2.4") @@ -113,6 +149,9 @@ configure(javaProjects) { dependency("org.wildfly.openssl:wildfly-openssl:1.0.7.Final") dependency("org.xerial.snappy:snappy-java:1.1.10.1") + // Arenadata encryption + dependency("io.arenadata.security:encryption:1.0.0") + // Hadoop dependencies dependencySet(group:"org.apache.hadoop", version:"${hadoopVersion}") { entry("hadoop-annotations") @@ -143,11 +182,13 @@ configure(javaProjects) { entry("hive-metastore") entry("hive-serde") entry("hive-common") + entry("hive-service") + entry("hive-service-rpc") } // 1.2.2 breaks on CDH-5.x - dependencySet(group:"org.apache.hive", version:"1.1.0") { + // We use custom hive-jdbc driver from Arenadata + dependencySet(group:"io.arenadata.hive", version:"2.3.8-arenadata-pxf-3") { entry("hive-jdbc") - entry("hive-service") } dependencySet(group:"org.apache.hive.shims", version:"${hiveVersion}") { entry("hive-shims-common") diff --git a/server/gradle.properties b/server/gradle.properties index 736f4a1a3c..2bdf9c0ade 100644 --- a/server/gradle.properties +++ b/server/gradle.properties @@ -25,7 +25,7 @@ hbaseVersion=1.3.2 junitVersion=4.11 parquetVersion=1.11.1 awsJavaSdk=1.12.261 -springBootVersion=2.7.12 +springBootVersion=2.4.3 org.gradle.daemon=true org.gradle.parallel=false orcVersion=1.6.13 diff --git a/server/pxf-api/build.gradle b/server/pxf-api/build.gradle index 64cc81b7d1..dd0578ec7b 100644 --- a/server/pxf-api/build.gradle +++ b/server/pxf-api/build.gradle @@ -39,6 +39,7 @@ dependencies { implementation("commons-configuration:commons-configuration") implementation("commons-lang:commons-lang") implementation("org.apache.commons:commons-lang3") + implementation("io.arenadata.security:encryption") implementation("org.apache.hadoop:hadoop-auth") { transitive = false } implementation("org.codehaus.woodstox:stax2-api") { transitive = false } diff --git a/server/pxf-api/src/main/java/org/greenplum/pxf/api/configuration/PxfJksTextEncryptorConfiguration.java b/server/pxf-api/src/main/java/org/greenplum/pxf/api/configuration/PxfJksTextEncryptorConfiguration.java new file mode 100644 index 0000000000..535504e70b --- /dev/null +++ b/server/pxf-api/src/main/java/org/greenplum/pxf/api/configuration/PxfJksTextEncryptorConfiguration.java @@ -0,0 +1,37 @@ +package org.greenplum.pxf.api.configuration; + +import io.arenadata.security.encryption.client.configuration.JksTextEncryptorConfiguration; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConditionalOnProperty({"pxf.ssl.jks-store.path", "pxf.ssl.jks-store.password", "pxf.ssl.salt.key"}) +public class PxfJksTextEncryptorConfiguration extends JksTextEncryptorConfiguration { + private final String path; + private final String password; + private final String key; + + public PxfJksTextEncryptorConfiguration(@Value("${pxf.ssl.jks-store.path}") String path, + @Value("${pxf.ssl.jks-store.password}") String password, + @Value("${pxf.ssl.salt.key}") String key) { + this.path = path; + this.password = password; + this.key = key; + } + + @Override + protected String jksStorePath() { + return path; + } + + @Override + protected char[] jksStorePassword() { + return password.toCharArray(); + } + + @Override + protected String secretKeyAlias() { + return key; + } +} diff --git a/server/pxf-api/src/main/java/org/greenplum/pxf/api/examples/DemoTextResolver.java b/server/pxf-api/src/main/java/org/greenplum/pxf/api/examples/DemoTextResolver.java index 775959e5ab..55eae11a5d 100644 --- a/server/pxf-api/src/main/java/org/greenplum/pxf/api/examples/DemoTextResolver.java +++ b/server/pxf-api/src/main/java/org/greenplum/pxf/api/examples/DemoTextResolver.java @@ -23,6 +23,8 @@ import org.greenplum.pxf.api.OneRow; import org.greenplum.pxf.api.io.DataType; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.util.LinkedList; import java.util.List; @@ -62,8 +64,16 @@ public OneRow setFields(List record) throws Exception { throw new Exception("Unexpected record format, expected 1 field, found " + (record == null ? 0 : record.size())); } - byte[] value = (byte[]) record.get(0).val; + int readCount; + byte[] data = new byte[1024]; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + DataInputStream dis = (DataInputStream) record.get(0).val; + while ((readCount = dis.read(data, 0, data.length)) != -1) { + buffer.write(data, 0, readCount); + } + buffer.flush(); + byte[] bytes= buffer.toByteArray(); // empty array means the end of input stream, return null to stop iterations - return value.length == 0 ? null : new OneRow(value); + return bytes.length == 0 ? null : new OneRow(bytes); } } diff --git a/server/pxf-api/src/main/java/org/greenplum/pxf/api/utilities/SpringContext.java b/server/pxf-api/src/main/java/org/greenplum/pxf/api/utilities/SpringContext.java index bd28a417b1..3b45216f7e 100644 --- a/server/pxf-api/src/main/java/org/greenplum/pxf/api/utilities/SpringContext.java +++ b/server/pxf-api/src/main/java/org/greenplum/pxf/api/utilities/SpringContext.java @@ -26,6 +26,14 @@ public static T getBean(Class requiredType) { return context.getBean(requiredType); } + public static T getNullableBean(Class requiredType) { + try { + return context.getBean(requiredType); + } catch (Exception e) { + return null; + } + } + @Override public void setApplicationContext(ApplicationContext context) throws BeansException { diff --git a/server/pxf-api/src/test/java/org/greenplum/pxf/api/DemoResolverTest.java b/server/pxf-api/src/test/java/org/greenplum/pxf/api/DemoResolverTest.java index 1f7e72fc5f..3e455a6ce9 100755 --- a/server/pxf-api/src/test/java/org/greenplum/pxf/api/DemoResolverTest.java +++ b/server/pxf-api/src/test/java/org/greenplum/pxf/api/DemoResolverTest.java @@ -26,6 +26,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -55,7 +57,7 @@ public void setup() { textResolver = new DemoTextResolver(); row = new OneRow("0.0", DATA); - field = new OneField(VARCHAR.getOID(), DATA.getBytes()); + field = new OneField(VARCHAR.getOID(), new DataInputStream(new ByteArrayInputStream(DATA.getBytes()))); } @Test @@ -79,7 +81,7 @@ public void testSetTextData() throws Exception { @Test public void testSetEmptyTextData() throws Exception { - OneField field = new OneField(VARCHAR.getOID(), new byte[]{}); + OneField field = new OneField(VARCHAR.getOID(), new DataInputStream(new ByteArrayInputStream(new byte[]{}))); OneRow output = textResolver.setFields(Collections.singletonList(field)); assertNull(output); } diff --git a/server/pxf-hdfs/src/test/java/org/greenplum/pxf/plugins/hdfs/AvroResolverTest.java b/server/pxf-hdfs/src/test/java/org/greenplum/pxf/plugins/hdfs/AvroResolverTest.java index b3fdcc5762..166698e8ae 100644 --- a/server/pxf-hdfs/src/test/java/org/greenplum/pxf/plugins/hdfs/AvroResolverTest.java +++ b/server/pxf-hdfs/src/test/java/org/greenplum/pxf/plugins/hdfs/AvroResolverTest.java @@ -711,7 +711,7 @@ private Schema getAvroSchemaForComplexTypes() { // add a RECORD with a float, int, and string inside fields.add(new Schema.Field( Schema.Type.RECORD.getName(), - createRecord(new Schema.Type[]{Schema.Type.FLOAT, Schema.Type.INT, Schema.Type.STRING}), + createRecord(new Schema.Type[]{Schema.Type.FLOAT, Schema.Type.INT, Schema.Type.STRING}), "", null) ); diff --git a/server/pxf-jdbc/README.md b/server/pxf-jdbc/README.md index e6ab2e2d0e..537473cfc1 100644 --- a/server/pxf-jdbc/README.md +++ b/server/pxf-jdbc/README.md @@ -102,7 +102,8 @@ User name (login) to use to connect to external database. #### JDBC password -Password to use to connect to external database. +Password to use to connect to external database. The password might be encrypted. +How to use encrypted password is described in section [JDBC password encryption](#jdbc-password-encryption). * **Option**: `PASS` * **Configuration parameter**: `jdbc.password` @@ -124,6 +125,35 @@ Whether PXF should quote column names when constructing SQL query to the externa When this setting is not set, PXF automatically checks whether some column name should be quoted, and if so, it quotes all column names in the query. +#### Convert to Oracle date type +*Can be set only in `LOCATION` clause of external table DDL. +It is used by only PXF Oracle JDBC driver for pushdown.* + +The parameter is used for some specific cases when you need to convert Postgres `timestamp` type to `date` type in Oracle for pushdown filter. + +* **Option**: `CONVERT_ORACLE_DATE` +* **Value**: + * not set — default value is `false`. Postgres `timestamp` type will be converted to Oracle `timestamp` type (default behaviour) + * `true` (case-insensitive) — convert Postgres `timestamp` type to Oracle `date` type + * any other value or `false` — Postgres `timestamp` type will be converted to Oracle `timestamp` type (default behaviour) + +If a field is `timestamp` type in the external GP table and `CONVERT_ORACLE_DATE=true` the fields that are used in the `where` filter will be cast to `date` type in Oracle. +The milliseconds will be truncated. Example of the query where c3 field has `timestamp` type in the GP and `date` type in the Oracle: +``` +query in gp: SELECT c1, c2, c3 FROM ext_oracle_datetime_fix WHERE c3 >= '2022-01-01 14:00:00.123456' and c3 < '2022-01-02 03:00:00.232323'; +recieved query in oracle: SELECT c1, c2, c3 FROM system.tst_pxf_datetime WHERE (c3 >= to_date('2022-01-01 14:00:00', 'YYYY-MM-DD HH24:MI:SS') AND c3 < to_date('2022-01-02 03:00:00', 'YYYY-MM-DD HH24:MI:SS')) +``` + +If the parameter `CONVERT_ORACLE_DATE=false` or it is not declared in the `LOCATION` the c3 field will be converted to `timestamp` type in Oracle (default behaviour): + +``` +query in gp: SELECT c1, c2, c3 FROM ext_oracle_datetime_fix where c3 >= '2022-01-01 12:01:00' and c3 < '2022-01-02 02:01:00'; +recieved query in oracle: SELECT c1, c2, c3 FROM system.tst_pxf_datetime WHERE (c3 >= to_timestamp('2022-01-01 12:01:00', 'YYYY-MM-DD HH24:MI:SS.FF') AND c3 < to_timestamp('2022-01-02 02:01:00', 'YYYY-MM-DD HH24:MI:SS.FF')) +``` + +**Notes:** +The parameter `CONVERT_ORACLE_DATE` has impact only on the fields that are used in the `where` filter and does not apply for the other fields with `timestamp` type in the table. + #### Partition by *Can be set only in `LOCATION` clause of external table DDL* @@ -621,3 +651,116 @@ Follow these steps to enable connectivity to Hive: If you enable impersonation, do not explicitly specify `hive.server2.proxy.user` property in the URL. - if Hive is configured with `hive.server2.enable.doAs = FALSE`, Hive will run Hadoop operations with the identity provided by the PXF Kerberos principal (usually `gpadmin`) + + +## JDBC password encryption +It is possible to use an encrypted password instead of the password in a paint text in the `$PXF_BASE/servers//jdbc-site.xml` file. + +### Prerequisites +There is a special library that is used to encrypt and decrypt password. The executable jar-file of this library has to be copied to `$PXF_BASE/lib/` directory on each segment. +It is used to encrypt password. The original jar-file of the library is used to decrypt password. It is added as a dependency to the PXF project. + +### How to enable encryption +Before using an encrypted password you have to **create keystore and add encryption key** to the store.\ +The keystore is a file where the encryption key will be saved. And the encryption key will be used to encrypt and decrypt password.\ +The keystore and the encryption key have to be created on each segment server. + +The command to create the keystore:\ +```keytool -keystore -storepass -genkey -keypass -alias ```, where\ +`keystore_file` - the file path of the keystore;\ +`keystore_password` - password which will be used to access the keystore;\ +`key_password` - password for the specific `keystore_alias`. It might be the same as `keystore_password`;\ +`keystore_alias` - name of the keystore. + +You will be asked to enter some information about your organization, first and last name, etc. after running the command. + +Example of the command to create a keystore:\ +`keytool -keystore /var/lib/pxf/conf/pxfkeystore.jks -storepass 12345678 -genkey -keypass 12345678 -alias pxfkeystore` + +The next step is to add encryption key.\ +The command to add encryption key to the keystore:\ +`keytool -keystore -storepass -importpass -keypass -alias `, where\ +`keystore_file` - the file path of the keystore that was created in the previous step;\ +`keystore_password` - password to access the keystore;\ +`key_password` - password for the specific `encryption_key_alias`. It might be the same as `keystore_password`;\ +`encryption_key_alias` - name of the encryption key. This name will be used to get encryption key from the keystore. + +You will be asked to enter an encryption key you want to store after running the command. + +Example of the command to add encryption key:\ +`keytool -keystore /var/lib/pxf/conf/pxfkeystore.jks -storepass 12345678 -importpass -keypass 12345678 -alias PXF_PASS_KEY`\ +*Enter the password to be stored:* qwerty + +In case of error `keytool error: java.security.KeyStoreException: Cannot store non-PrivateKeys` run the following command before adding an encryption key:\ +`keytool -importkeystore -srckeystore -destkeystore -deststoretype pkcs12`, where\ +`` - the file path of the keystore which has been created before;\ +Example of the command:\ +`keytool -importkeystore -srckeystore /var/lib/pxf/conf/pxfkeystore.jks -destkeystore /var/lib/pxf/conf/pxfkeystore.jks -deststoretype pkcs12` + + +Next, additional properties have to be added into the `$PXF_BASE/conf/pxf-application.properties` file on each segment:\ +`pxf.ssl.jks-store.path` - a Java keystore (JKS) absolute file path. It is a `keystore_file` from the command to create the keystore;\ +`pxf.ssl.jks-store.password` - a Java keystore password. It is a `keystore_password` from the command to create the keystore;\ +`pxf.ssl.salt.key` - an alias which is used to get encryption key from the keystore. It is an `encryption_key_alias` from the command to add encryption key to the keystore. + +You have to restart PXF service after adding the properties. + +Example of the properties in the `pxf-application.properties` file: +``` +# Encryption +pxf.ssl.jks-store.path=/var/lib/pxf/conf/pxfkeystore.jks +pxf.ssl.jks-store.password=12345678 +pxf.ssl.salt.key=PXF_PASS_KEY +``` + +### How to use encryption +The first step is to encrypt password that is used to connect to the database.\ +There is a special command to do this action:\ +`pxf encrypt `, where\ +`` - password in a plain text that is used to connect to the database. This password will be encrypted;\ +`` - Optional. The algorithm to encrypt password. Default value: `aes256` + +The result of the command will be an encrypted password in a format `aes256:encrypted_password` + +Example of the command to encrypt password:\ +`pxf encrypt biuserpassword`\ +*Output:* aes256:7BhhI+10ut+xM70iRlyxVDD/tokap3pbK2bmkLgPOYLH7NcfEYJSAIYkApjKM3Zu + +Next, you have to copy the encrypted password including aes256 prefix and paste it into `$PXF_BASE/servers//jdbc-site.xml` file +instead of the password in a plain text. + +The example of the `jdbc-site.xml` with encrypted password: +```xml + + + + jdbc.driver + org.postgresql.Driver + Class name of the JDBC driver (e.g. org.postgresql.Driver) + + + jdbc.url + jdbc:postgresql://10.10.10.20/adb + The URL that the JDBC driver can use to connect to the database (e.g. jdbc:postgresql://localhost/postgres) + + + jdbc.user + bi_user + User name for connecting to the database (e.g. postgres) + + + jdbc.password + aes256:7BhhI+10ut+xM70iRlyxVDD/tokap3pbK2bmkLgPOYLH7NcfEYJSAIYkApjKM3Zu + Password for connecting to the database (e.g. postgres) + + +``` + +You don't need to make any additional changes when you crate an external table. The decryption engine will recognize whether the password is encrypted or not. +If the password is encrypted the decrypter will take care about the password. If the password is in plain text format it will be passed as is to the JDBC connection manager. + + + + + + diff --git a/server/pxf-jdbc/build.gradle b/server/pxf-jdbc/build.gradle index 9d050cf2de..268987435e 100644 --- a/server/pxf-jdbc/build.gradle +++ b/server/pxf-jdbc/build.gradle @@ -31,13 +31,15 @@ dependencies { * Transitive Dependencies for JDBC Hive Access *******************************/ - implementation("org.apache.hive:hive-jdbc") { transitive = false } + implementation("io.arenadata.hive:hive-jdbc") { transitive = false } implementation("org.apache.hive:hive-service") { transitive = false } + implementation("org.apache.hive:hive-service-rpc") { transitive = false } implementation("org.apache.thrift:libthrift") { transitive = false } implementation("org.apache.hive:hive-common") { transitive = false } implementation("org.apache.hive.shims:hive-shims-0.23") { transitive = false } implementation("org.apache.hive.shims:hive-shims-common") { transitive = false } implementation("org.springframework.boot:spring-boot-starter-log4j2") + implementation("io.arenadata.security:encryption") /******************************* * Test Dependencies diff --git a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcAccessor.java b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcAccessor.java index ab883f8fcd..e6eb85a33b 100644 --- a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcAccessor.java +++ b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcAccessor.java @@ -19,6 +19,7 @@ * under the License. */ +import io.arenadata.security.encryption.client.service.DecryptClient; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.greenplum.pxf.api.OneRow; @@ -84,8 +85,8 @@ public JdbcAccessor() { * @param connectionManager connection manager * @param secureLogin the instance of the secure login */ - JdbcAccessor(ConnectionManager connectionManager, SecureLogin secureLogin) { - super(connectionManager, secureLogin); + JdbcAccessor(ConnectionManager connectionManager, SecureLogin secureLogin, DecryptClient decryptClient) { + super(connectionManager, secureLogin, decryptClient); } /** @@ -122,6 +123,11 @@ private boolean openForReadInner(Connection connection) throws SQLException { } else if (quoteColumns) { sqlQueryBuilder.forceSetQuoteString(); } + + if (wrapDateWithTime) { + sqlQueryBuilder.setWrapDateWithTime(true); + } + // Read variables String queryRead = sqlQueryBuilder.buildSelectQuery(); LOG.trace("Select query: {}", queryRead); diff --git a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePlugin.java b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePlugin.java index eda0062ebf..3e88ec579d 100644 --- a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePlugin.java +++ b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePlugin.java @@ -19,6 +19,7 @@ * under the License. */ +import io.arenadata.security.encryption.client.service.DecryptClient; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.greenplum.pxf.api.model.BasePlugin; @@ -139,6 +140,9 @@ public static TransactionIsolation typeOf(String str) { // Query timeout. protected Integer queryTimeout; + // Convert Postgres timestamp to Oracle date with time + protected boolean wrapDateWithTime = false; + // Quote columns setting set by user (three values are possible) protected Boolean quoteColumns = null; @@ -164,6 +168,7 @@ public static TransactionIsolation typeOf(String str) { private final ConnectionManager connectionManager; private final SecureLogin secureLogin; + private final DecryptClient decryptClient; // Flag which is used when the year might contain more than 4 digits in `date` or 'timestamp' protected boolean isDateWideRange; @@ -177,10 +182,12 @@ public static TransactionIsolation typeOf(String str) { /** * Creates a new instance with default (singleton) instances of - * ConnectionManager and SecureLogin. + * ConnectionManager, SecureLogin and DecryptClient. */ JdbcBasePlugin() { - this(SpringContext.getBean(ConnectionManager.class), SpringContext.getBean(SecureLogin.class)); + this(SpringContext.getBean(ConnectionManager.class), SpringContext.getBean(SecureLogin.class), + SpringContext.getNullableBean(DecryptClient.class) + ); } /** @@ -188,9 +195,10 @@ public static TransactionIsolation typeOf(String str) { * * @param connectionManager connection manager instance */ - JdbcBasePlugin(ConnectionManager connectionManager, SecureLogin secureLogin) { + JdbcBasePlugin(ConnectionManager connectionManager, SecureLogin secureLogin, DecryptClient decryptClient) { this.connectionManager = connectionManager; this.secureLogin = secureLogin; + this.decryptClient = decryptClient; } @Override @@ -261,6 +269,12 @@ public void afterPropertiesSet() { } } + // Optional parameter. The default value is false + String wrapDateWithTimeRaw = context.getOption("CONVERT_ORACLE_DATE"); + if (wrapDateWithTimeRaw != null) { + wrapDateWithTime = Boolean.parseBoolean(wrapDateWithTimeRaw); + } + // Optional parameter. The default value is null String quoteColumnsRaw = context.getOption("QUOTE_COLUMNS"); if (quoteColumnsRaw != null) { @@ -335,6 +349,12 @@ public void afterPropertiesSet() { if (jdbcUser != null) { String jdbcPassword = configuration.get(JDBC_PASSWORD_PROPERTY_NAME); if (jdbcPassword != null) { + try { + jdbcPassword = decryptClient == null ? jdbcPassword : decryptClient.decrypt(jdbcPassword); + } catch (Exception e) { + throw new RuntimeException( + "Failed to decrypt jdbc password. " + e.getMessage(), e); + } LOG.debug("Connection password: {}", ConnectionManager.maskPassword(jdbcPassword)); connectionConfiguration.setProperty("password", jdbcPassword); } diff --git a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcPredicateBuilder.java b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcPredicateBuilder.java index c5c9f4cc0a..64be4ceb42 100644 --- a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcPredicateBuilder.java +++ b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/JdbcPredicateBuilder.java @@ -39,6 +39,7 @@ public class JdbcPredicateBuilder extends ColumnPredicateBuilder { private final DbProduct dbProduct; + private boolean wrapDateWithTime = false; public JdbcPredicateBuilder(DbProduct dbProduct, List tupleDescription) { @@ -52,6 +53,15 @@ public JdbcPredicateBuilder(DbProduct dbProduct, this.dbProduct = dbProduct; } + public JdbcPredicateBuilder(DbProduct dbProduct, + String quoteString, + List tupleDescription, + boolean wrapDateWithTime) { + super(quoteString, tupleDescription); + this.dbProduct = dbProduct; + this.wrapDateWithTime = wrapDateWithTime; + } + @Override public String toString() { StringBuilder sb = getStringBuilder(); @@ -79,7 +89,12 @@ protected String serializeValue(DataType type, String value) { return dbProduct.wrapDate(value); case TIMESTAMP: // Timestamp field has different format in different databases - return dbProduct.wrapTimestamp(value); + // If wrapDateWithTime = true we have to convert timestamp to Oracle `date with time` + if (wrapDateWithTime) { + return dbProduct.wrapDateWithTime(value); + } else { + return dbProduct.wrapTimestamp(value); + } default: throw new UnsupportedOperationException(String.format( "Unsupported column type for filtering '%s' ", type.getOID())); diff --git a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/SQLQueryBuilder.java b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/SQLQueryBuilder.java index 1960e23563..a472305a73 100644 --- a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/SQLQueryBuilder.java +++ b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/SQLQueryBuilder.java @@ -58,8 +58,7 @@ public class SQLQueryBuilder { Operator.EQUALS, Operator.LIKE, Operator.NOT_EQUALS, - // TODO: In is not supported? - // Operator.IN, + Operator.IN, Operator.IS_NULL, Operator.IS_NOT_NULL, Operator.NOOP, @@ -77,6 +76,7 @@ public class SQLQueryBuilder { private final List columns; private final String source; private String quoteString; + private boolean wrapDateWithTime = false; private boolean subQueryUsed = false; /** @@ -122,6 +122,10 @@ public SQLQueryBuilder(RequestContext context, DatabaseMetaData metaData, String quoteString = ""; } + public void setWrapDateWithTime(boolean wrapDateWithTime) { + this.wrapDateWithTime = wrapDateWithTime; + } + /** * Build SELECT query (with "WHERE" and partition constraints). * @@ -139,7 +143,9 @@ public String buildSelectQuery() { // Insert partition constraints buildFragmenterSql(context, dbProduct, quoteString, sb); - return sb.toString(); + String query = sb.toString(); + LOG.debug("buildSelectQuery: {}", query); + return query; } /** @@ -268,7 +274,8 @@ protected JdbcPredicateBuilder getPredicateBuilder() { return new JdbcPredicateBuilder( dbProduct, quoteString, - context.getTupleDescription()); + context.getTupleDescription(), + wrapDateWithTime); } /** @@ -287,18 +294,24 @@ protected TreeVisitor getPruner() { * @param query SQL query to insert constraints to. The query may may contain other WHERE statements */ private void buildWhereSQL(StringBuilder query) { - if (!context.hasFilter()) return; + if (!context.hasFilter()) { + LOG.debug("FILTER empty"); + return; + } JdbcPredicateBuilder jdbcPredicateBuilder = getPredicateBuilder(); try { // Parse the filter string into a expression tree Node Node root = new FilterParser().parse(context.getFilterString()); + LOG.debug("FILTER source: {}", context.getFilterString()); // Prune the parsed tree with the provided pruner and then // traverse the tree with the JDBC predicate builder to produce a predicate TRAVERSER.traverse(root, getPruner(), jdbcPredicateBuilder); // No exceptions were thrown, change the provided query - query.append(jdbcPredicateBuilder.toString()); + String where = jdbcPredicateBuilder.toString(); + LOG.debug("FILTER target: {}", where); + query.append(where); } catch (Exception e) { LOG.debug("WHERE clause is omitted: " + e.toString()); // Silence the exception and do not insert constraints diff --git a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/utils/DbProduct.java b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/utils/DbProduct.java index 7e3d6a5bbe..7ab1d55914 100644 --- a/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/utils/DbProduct.java +++ b/server/pxf-jdbc/src/main/java/org/greenplum/pxf/plugins/jdbc/utils/DbProduct.java @@ -33,6 +33,11 @@ public String wrapDate(Object val) { return "'" + val + "'"; } + @Override + public String wrapDateWithTime(Object val) { + return wrapTimestamp(val); + } + @Override public String buildSessionQuery(String key, String value) { return String.format("SET %s %s", key, value); @@ -44,6 +49,11 @@ public String buildSessionQuery(String key, String value) { public String wrapDate(Object val) { return "DATE('" + val + "')"; } + + @Override + public String wrapDateWithTime(Object val) { + return wrapTimestamp(val); + } }, ORACLE { @@ -52,6 +62,16 @@ public String wrapDate(Object val) { return "to_date('" + val + "', 'YYYY-MM-DD')"; } + @Override + public String wrapDateWithTime(Object val) { + String valStr = String.valueOf(val); + int index = valStr.lastIndexOf('.'); + if (index != -1) { + valStr = valStr.substring(0, index); + } + return "to_date('" + valStr + "', 'YYYY-MM-DD HH24:MI:SS')"; + } + @Override public String wrapTimestamp(Object val) { return "to_timestamp('" + val + "', 'YYYY-MM-DD HH24:MI:SS.FF')"; @@ -68,6 +88,11 @@ public String buildSessionQuery(String key, String value) { public String wrapDate(Object val) { return "date'" + val + "'"; } + + @Override + public String wrapDateWithTime(Object val) { + return wrapTimestamp(val); + } }, S3_SELECT { @@ -76,10 +101,30 @@ public String wrapDate(Object val) { return "TO_TIMESTAMP('" + val + "')"; } + @Override + public String wrapDateWithTime(Object val) { + return wrapTimestamp(val); + } + @Override public String wrapTimestamp(Object val) { return "TO_TIMESTAMP('" + val + "')"; } + }, + + SYBASE { + @Override + public String wrapDate(Object val) { return "'" + val + "'"; } + + @Override + public String wrapDateWithTime(Object val) { + return wrapTimestamp(val); + } + + @Override + public String buildSessionQuery(String key, String value) { + return String.format("SET %s %s", key, value); + } }; /** @@ -90,6 +135,15 @@ public String wrapTimestamp(Object val) { */ public abstract String wrapDate(Object val); + /** + * Wraps a given date value to the date with time. + * It might be used in some special cases. + * + * @param val {@link java.sql.Date} object to wrap + * @return a string with a properly wrapped date object + */ + public abstract String wrapDateWithTime(Object val); + /** * Wraps a given timestamp value the way required by target database * @@ -132,6 +186,8 @@ else if (dbName.contains("ORACLE")) result = DbProduct.ORACLE; else if (dbName.contains("S3 SELECT")) result = DbProduct.S3_SELECT; + else if (dbName.contains("ADAPTIVE SERVER ENTERPRISE")) + result = DbProduct.SYBASE; else result = DbProduct.POSTGRES; diff --git a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcAccessorTest.java b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcAccessorTest.java index 6e3ea1ed82..a443a2563a 100644 --- a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcAccessorTest.java +++ b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcAccessorTest.java @@ -1,5 +1,6 @@ package org.greenplum.pxf.plugins.jdbc; +import io.arenadata.security.encryption.client.service.DecryptClient; import org.apache.hadoop.conf.Configuration; import org.greenplum.pxf.api.error.PxfRuntimeException; import org.greenplum.pxf.api.model.RequestContext; @@ -52,11 +53,13 @@ public class JdbcAccessorTest { private PreparedStatement mockPreparedStatement; @Mock private ResultSet mockResultSet; + @Mock + private DecryptClient mockDecryptClient; @BeforeEach public void setup() { - accessor = new JdbcAccessor(mockConnectionManager, mockSecureLogin); + accessor = new JdbcAccessor(mockConnectionManager, mockSecureLogin, mockDecryptClient); configuration = new Configuration(); context = new RequestContext(); context.setConfig("default"); diff --git a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePluginTest.java b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePluginTest.java index 63cbefd2af..924fd5bf5a 100644 --- a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePluginTest.java +++ b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePluginTest.java @@ -19,6 +19,7 @@ * under the License. */ +import io.arenadata.security.encryption.client.service.DecryptClient; import org.apache.hadoop.conf.Configuration; import org.greenplum.pxf.api.model.RequestContext; import org.greenplum.pxf.api.security.SecureLogin; @@ -65,6 +66,8 @@ public class JdbcBasePluginTest { private PreparedStatement mockStatement; @Mock private SecureLogin mockSecureLogin; + @Mock + private DecryptClient mockDecryptClient; private final SQLException exception = new SQLException("some error"); private Configuration configuration; @@ -239,7 +242,7 @@ public void testTransactionIsolationNotSetByUser() throws SQLException { when(mockConnectionManager.getConnection(any(), any(), any(), anyBoolean(), any(), any())).thenReturn(mockConnection); when(mockConnection.getMetaData()).thenReturn(mockMetaData); - JdbcBasePlugin plugin = new JdbcBasePlugin(mockConnectionManager, mockSecureLogin); + JdbcBasePlugin plugin = new JdbcBasePlugin(mockConnectionManager, mockSecureLogin, mockDecryptClient); plugin.setRequestContext(context); Connection conn = plugin.getConnection(); @@ -298,7 +301,7 @@ public void testTransactionIsolationSetByUserFailedToGetMetadata() throws SQLExc when(mockConnectionManager.getConnection(anyString(), anyString(), any(), anyBoolean(), any(), anyString())).thenReturn(mockConnection); doThrow(new SQLException("")).when(mockConnection).getMetaData(); - JdbcBasePlugin plugin = new JdbcBasePlugin(mockConnectionManager, mockSecureLogin); + JdbcBasePlugin plugin = new JdbcBasePlugin(mockConnectionManager, mockSecureLogin, mockDecryptClient); plugin.setRequestContext(context); assertThrows(SQLException.class, plugin::getConnection); } @@ -324,7 +327,7 @@ public void testGetPreparedStatementDoesNotSetQueryTimeoutIfNotSpecified() throw when(mockConnection.prepareStatement(anyString())).thenReturn(mockStatement); - JdbcBasePlugin plugin = new JdbcBasePlugin(mockConnectionManager, mockSecureLogin); + JdbcBasePlugin plugin = new JdbcBasePlugin(mockConnectionManager, mockSecureLogin, mockDecryptClient); plugin.setRequestContext(context); plugin.getPreparedStatement(mockConnection, "foo"); @@ -495,7 +498,7 @@ public void testDateWideRangeFromConfiguration() throws SQLException { } private JdbcBasePlugin getPlugin(ConnectionManager mockConnectionManager, SecureLogin mockSecureLogin, RequestContext context) { - JdbcBasePlugin plugin = new JdbcBasePlugin(mockConnectionManager, mockSecureLogin); + JdbcBasePlugin plugin = new JdbcBasePlugin(mockConnectionManager, mockSecureLogin, mockDecryptClient); plugin.setRequestContext(context); plugin.afterPropertiesSet(); return plugin; diff --git a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePluginTestInitialize.java b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePluginTestInitialize.java index 6b0cdac06d..185cc8bbe8 100644 --- a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePluginTestInitialize.java +++ b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/JdbcBasePluginTestInitialize.java @@ -20,6 +20,8 @@ */ import com.google.common.base.Ticker; +import io.arenadata.security.encryption.client.provider.TextEncryptorProvider; +import io.arenadata.security.encryption.client.service.impl.DecryptClientImpl; import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.PxfUserGroupInformation; @@ -89,7 +91,8 @@ public void setup() { ); PxfUserGroupInformation mockPxfUserGroupInformation = mock(PxfUserGroupInformation.class); - plugin = new JdbcBasePlugin(connectionManager, new SecureLogin(mockPxfUserGroupInformation)); + TextEncryptorProvider mockTextEncryptorProvider = mock(TextEncryptorProvider.class); + plugin = new JdbcBasePlugin(connectionManager, new SecureLogin(mockPxfUserGroupInformation), new DecryptClientImpl(mockTextEncryptorProvider)); } /** diff --git a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/SQLQueryBuilderTest.java b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/SQLQueryBuilderTest.java index 5c1562a130..2e3a468bc4 100644 --- a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/SQLQueryBuilderTest.java +++ b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/SQLQueryBuilderTest.java @@ -35,6 +35,7 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -133,17 +134,59 @@ public void testIsNotNullOperator() throws Exception { } @Test - public void testUnsupportedOperationFilter() throws Exception { + public void testInOperatorWithWhere() throws Exception { when(mockMetaData.getDatabaseProductName()).thenReturn("mysql"); when(mockMetaData.getExtraNameCharacters()).thenReturn(""); - // IN 'bad' - context.setFilterString("a3c25s3dbado10"); + // grade IN ('bad') + context.setFilterString("a3m1009s3dbado10"); SQLQueryBuilder builder = new SQLQueryBuilder(context, mockMetaData); builder.autoSetQuoteString(); String query = builder.buildSelectQuery(); - assertEquals(SQL, query); + assertEquals(SQL + " WHERE grade IN ('bad')", query); + } + + @Test + public void testInOperatorWithWhereAndFewIn() throws Exception { + when(mockMetaData.getDatabaseProductName()).thenReturn("mysql"); + when(mockMetaData.getExtraNameCharacters()).thenReturn(""); + + // grade IN ('bad','good') + context.setFilterString("a3m1009s3dbads4dgoodo10"); + + SQLQueryBuilder builder = new SQLQueryBuilder(context, mockMetaData); + builder.autoSetQuoteString(); + String query = builder.buildSelectQuery(); + assertEquals(SQL + " WHERE grade IN ('bad','good')", query); + } + + @Test + public void testInOperatorShouldContainWhere() throws Exception { + when(mockMetaData.getDatabaseProductName()).thenReturn("mysql"); + when(mockMetaData.getExtraNameCharacters()).thenReturn(""); + + // grade IN ('bad') + context.setFilterString("a3m1009s3dbado10"); + + SQLQueryBuilder builder = new SQLQueryBuilder(context, mockMetaData); + builder.autoSetQuoteString(); + String query = builder.buildSelectQuery(); + assertNotEquals(SQL, query); + } + + @Test + public void testInOperatorShouldContainBrackets() throws Exception { + when(mockMetaData.getDatabaseProductName()).thenReturn("mysql"); + when(mockMetaData.getExtraNameCharacters()).thenReturn(""); + + // grade IN ('bad') + context.setFilterString("a3m1009s3dbado10"); + + SQLQueryBuilder builder = new SQLQueryBuilder(context, mockMetaData); + builder.autoSetQuoteString(); + String query = builder.buildSelectQuery(); + assertNotEquals(SQL + " WHERE grade IN 'bad'", query); } @Test diff --git a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/utils/DbProductTest.java b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/utils/DbProductTest.java index 517cfac3df..cf859cfecf 100644 --- a/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/utils/DbProductTest.java +++ b/server/pxf-jdbc/src/test/java/org/greenplum/pxf/plugins/jdbc/utils/DbProductTest.java @@ -96,6 +96,17 @@ public void testOracleDates() { } } + @Test + public void testOracleDatesWithTime() { + final String[] expected = {"to_date('2001-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS')"}; + + DbProduct dbProduct = DbProduct.getDbProduct(DB_NAME_ORACLE); + + for (int i = 0; i < TIMESTAMPS.length; i++) { + assertEquals(expected[i], dbProduct.wrapDateWithTime(TIMESTAMPS[i])); + } + } + @Test public void testOracleTimestamps() { final String[] expected = {"to_timestamp('2001-01-01 00:00:00.0', 'YYYY-MM-DD HH24:MI:SS.FF')"}; diff --git a/server/pxf-service/build.gradle b/server/pxf-service/build.gradle index 54953ec339..db1dfda931 100644 --- a/server/pxf-service/build.gradle +++ b/server/pxf-service/build.gradle @@ -39,12 +39,13 @@ dependencies { implementation("org.apache.logging.log4j:log4j-spring-boot") implementation('org.springframework.boot:spring-boot-starter-actuator') implementation('io.micrometer:micrometer-registry-prometheus') + implementation('org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:3.0.6') + implementation('org.mockito:mockito-inline') implementation("com.univocity:univocity-parsers") implementation("org.apache.hadoop:hadoop-hdfs-client") { transitive = false } implementation("org.apache.hadoop:hadoop-auth") { transitive = false } - /******************************* * These JARs below (and its transitive dependencies, other than txw2 [for writing XML docs]) are needed for Java 11 * jcenter doesn't have full com.sun.xml.bind:jaxb-core/jaxb-impl packages, using glassfish distro diff --git a/server/pxf-service/src/main/java/org/greenplum/pxf/service/rest/ServiceMetricsRestController.java b/server/pxf-service/src/main/java/org/greenplum/pxf/service/rest/ServiceMetricsRestController.java new file mode 100644 index 0000000000..892ea2262e --- /dev/null +++ b/server/pxf-service/src/main/java/org/greenplum/pxf/service/rest/ServiceMetricsRestController.java @@ -0,0 +1,81 @@ +package org.greenplum.pxf.service.rest; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.actuate.metrics.MetricsEndpoint; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Objects; +import java.util.stream.Collectors; + +@RestController +@RequestMapping("/service-metrics") +public class ServiceMetricsRestController { + private static final Collection METRIC_NAMES = Arrays.asList( + "http.server.requests", + "jvm.buffer.count", + "jvm.buffer.memory.used", + "jvm.buffer.total.capacity", + "jvm.memory.committed", + "jvm.memory.max", + "jvm.memory.used", + "pxf.bytes.received", + "pxf.bytes.sent", + "pxf.fragments.sent", + "pxf.records.received", + "pxf.records.sent" + ); + private static final Collection CLUSTER_METRIC_NAMES = Arrays.asList( + "jvm.memory.committed", + "jvm.memory.max", + "jvm.memory.used", + "pxf.bytes.received", + "pxf.bytes.sent", + "pxf.records.received", + "pxf.records.sent" + ); + private final MetricsEndpoint metricsEndpoint; + private final String clusterName; + private final String hostName; + + public ServiceMetricsRestController(final MetricsEndpoint metricsEndpoint, + @Value("${cluster-name}") final String clusterName, + @Value("${eureka.instance.hostname}") final String hostName) { + this.metricsEndpoint = metricsEndpoint; + this.clusterName = clusterName; + this.hostName = hostName; + } + + @GetMapping + public Collection get() { + return METRIC_NAMES.stream() + .map(name -> metricsEndpoint.metric(name, Collections.emptyList())) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + @GetMapping("/cluster-metrics") + public ClusterMetrics getClusterMetrics() { + return new ClusterMetrics( + clusterName, + hostName, + CLUSTER_METRIC_NAMES.stream() + .map(name -> metricsEndpoint.metric(name, Collections.emptyList())) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + } + + @Getter + @RequiredArgsConstructor + public static class ClusterMetrics { + private final String cluster; + private final String hostname; + private final Collection metrics; + } +} diff --git a/server/pxf-service/src/main/resources/application.properties b/server/pxf-service/src/main/resources/application.properties index 2954c591b8..97316c822d 100644 --- a/server/pxf-service/src/main/resources/application.properties +++ b/server/pxf-service/src/main/resources/application.properties @@ -66,3 +66,18 @@ logging.file.path=${pxf.logdir:/tmp} # as query options. The default value is empty. # Used by FDW test cases that setup this property for test:* profiles and custom test Fragmenters / Accessors / Resolvers pxf.profile.dynamic.regex= + +cluster-name=${cluster_name:} + +# eureka +eureka.client.enabled=${adcc_enabled:false} +eureka.client.service-url.defaultZone=${ADCC_EUREKA_CLIENT_SERV_URL_DEF_ZONE:http://0.0.0.0:8761/eureka} +eureka.instance.prefer-ip-address=${ADCC_EUREKA_CLIENT_PREFER_IP_ADDRESS:true} +eureka.instance.hostname=${pxf_hostname:localhost} +eureka.instance.ip-address=${pxf_ip_address:127.0.0.1} +eureka.instance.appname=PXF SERVICE +eureka.instance.metadata-map.port=${server.port} +eureka.instance.metadata-map.cluster=${cluster-name} +eureka.instance.metadata-map.name=pxf-service +eureka.instance.metadata-map.version=${pxf-version:0.0.0-SNAPSHOT} +eureka.instance.metadata-map.status=UP \ No newline at end of file diff --git a/server/pxf-service/src/scripts/pxf b/server/pxf-service/src/scripts/pxf index f0c363b40e..51c880c43b 100755 --- a/server/pxf-service/src/scripts/pxf +++ b/server/pxf-service/src/scripts/pxf @@ -222,6 +222,7 @@ function doHelp() { It creates the servers, logs, lib, keytabs, and run directories inside \$PXF_BASE and copies configuration files. migrate migrates configurations from older installations of PXF + encrypt encrypt password with specified encryptor type. Default encryptor type is aes256 cluster perform on all the segment hosts in the cluster; try ${bold}pxf cluster help$normal sync synchronize \$PXF_BASE/{conf,lib,servers} directories onto . Use --delete to delete extraneous remote files @@ -475,6 +476,34 @@ function doSync() { rsync -az${DELETE:+ --delete} -e "ssh -o StrictHostKeyChecking=no" "$PXF_BASE"/{conf,lib,servers} "${target_host}:$PXF_BASE" } +function doEncrypt() { + local pwd=$1 + local encryptorType=$2 + if [[ -z $pwd ]]; then + fail 'Please provide password you want to encrypt' + fi + if [[ -z $encryptorType ]]; then + encryptorType=aes256 + fi + conf_file="$PXF_BASE/conf/pxf-application.properties" + if [[ -z $conf_file ]]; then + fail "File 'pxf-application.properties' was not found in PXF_BASE/conf/ directory" + fi + encr_jar_file=$(find "$PXF_BASE/lib" -maxdepth 1 -type f -name 'encryption-*.jar') + if [[ -z $encr_jar_file ]]; then + fail "Encryption library was not found in $PXF_BASE/lib/ directory" + fi + jksPath=$(getProperty 'pxf.ssl.jks-store.path' "$conf_file") + jksPassword=$(getProperty 'pxf.ssl.jks-store.password' "$conf_file") + jksEncryptKeyAlias=$(getProperty 'pxf.ssl.salt.key' "$conf_file") + checkJavaHome + java -jar $encr_jar_file -command encrypt -jksPath $jksPath -jksPassword $jksPassword -jksEncryptKeyAlias $jksEncryptKeyAlias -message $pwd -encryptorType $encryptorType +} + +function getProperty { + grep "^${1}" ${2} | cut -d'=' -f2 +} + function doCluster() { local cmd=$2 # Go CLI handles unset PXF_BASE when appropriate @@ -526,6 +555,9 @@ case $pxf_script_command in doSync "$2" fi ;; + 'encrypt') + doEncrypt "$2" "$3" + ;; 'help' | '-h' | '--help') doHelp ;; diff --git a/server/pxf-service/src/test/java/org/greenplum/pxf/service/rest/ServiceMetricsRestControllerTest.java b/server/pxf-service/src/test/java/org/greenplum/pxf/service/rest/ServiceMetricsRestControllerTest.java new file mode 100644 index 0000000000..3943cf0ebf --- /dev/null +++ b/server/pxf-service/src/test/java/org/greenplum/pxf/service/rest/ServiceMetricsRestControllerTest.java @@ -0,0 +1,40 @@ +package org.greenplum.pxf.service.rest; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.boot.actuate.metrics.MetricsEndpoint; + +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class ServiceMetricsRestControllerTest { + private static final String clusterName = "cluster name"; + private static final String hostName = "host name"; + private final MetricsEndpoint metricsEndpoint = mock(MetricsEndpoint.class); + private final ServiceMetricsRestController controller = new ServiceMetricsRestController(metricsEndpoint, clusterName, hostName); + + @Test + public void get() { + MetricsEndpoint.MetricResponse metricResponse = mock(MetricsEndpoint.MetricResponse.class); + when(metricsEndpoint.metric(anyString(), any())).thenReturn(metricResponse).thenReturn(null); + Collection result = controller.get(); + assertEquals(1, result.size()); + result.stream().findAny().filter(metricResponse::equals).orElseGet(Assertions::fail); + } + + @Test + public void getClusterMetrics() { + MetricsEndpoint.MetricResponse metricResponse = mock(MetricsEndpoint.MetricResponse.class); + when(metricsEndpoint.metric(anyString(), any())).thenReturn(metricResponse).thenReturn(null); + ServiceMetricsRestController.ClusterMetrics result = controller.getClusterMetrics(); + assertEquals(clusterName, result.getCluster()); + assertEquals(hostName, result.getHostname()); + assertEquals(1, result.getMetrics().size()); + result.getMetrics().stream().findAny().filter(metricResponse::equals).orElseGet(Assertions::fail); + } +} \ No newline at end of file