Setup extension TLS

Docs/CERTIFICATE_GENERATION.md

@@ -0,0 +1,137 @@
+# Certificate Generation
+
+Use the script below to generate certificates for running an extension. 
+The certificate for the extension needs to be in the same chain as the 
+root certificate for the OpenSearch cluster defined in the `plugins.security.ssl.http.pemtrustedcas_filepath`
+setting of `opensearch.yml`
+
+
+## Certificate Generation Script
+
+```
+#! /bin/bash
+
+openssl genrsa -out root-ca-key.pem 2048
+openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "/C=US/ST=NEW YORK/L=BROOKLYN/O=OPENSEARCH/OU=SECURITY/CN=ROOT" -out root-ca.pem -days 730
+
+openssl genrsa -out extension-01-key-temp.pem 2048
+openssl pkcs8 -inform PEM -outform PEM -in extension-01-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out extension-01-key.pem
+openssl req -new -key extension-01-key.pem -subj "/C=US/ST=NEW YORK/L=BROOKLYN/O=OPENSEARCH/OU=SECURITY/CN=extension-01" -out extension-01.csr
+echo 'subjectAltName=DNS:extension-01' | tee -a extension-01.ext
+echo 'subjectAltName=IP:172.20.0.11' | tee -a extension-01.ext
+openssl x509 -req -in extension-01.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out extension-01.pem -days 730 -extfile extension-01.ext
+
+rm extension-01-key-temp.pem
+rm extension-01.csr
+rm extension-01.ext
+rm root-ca.srl
+```
+
+## Certificate Generation Script for OpenSearch with single node and admin cert
+
+```
+#! /bin/bash
+
+openssl genrsa -out admin-key-temp.pem 2048
+openssl pkcs8 -inform PEM -outform PEM -in admin-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out admin-key.pem
+openssl req -new -key admin-key.pem -subj "/C=US/ST=NEW YORK/L=BROOKLYN/O=OPENSEARCH/OU=SECURITY/CN=A" -out admin.csr
+openssl x509 -req -in admin.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out admin.pem -days 730
+openssl genrsa -out os-node-01-key-temp.pem 2048
+openssl pkcs8 -inform PEM -outform PEM -in os-node-01-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out os-node-01-key.pem
+openssl req -new -key os-node-01-key.pem -subj "/C=US/ST=NEW YORK/L=BROOKLYN/O=OPENSEARCH/OU=SECURITY/CN=os-node-01" -out os-node-01.csr
+echo 'subjectAltName=DNS:os-node-01' | tee -a os-node-01.ext
+echo 'subjectAltName=IP:172.20.0.11' | tee -a os-node-01.ext
+openssl x509 -req -in os-node-01.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out os-node-01.pem -days 730 -extfile os-node-01.ext
+
+rm admin-key-temp.pem
+rm admin.csr
+rm os-node-01-key-temp.pem
+rm os-node-01.csr
+rm os-node-01.ext
+rm root-ca.srl
+```
+
+## Install Security plugin and run in SSL only mode
+
+To test an extension running with SSL and connected to an OpenSearch node with SSL you must install
+the security plugin in the OpenSearch node and run it in SSL only mode.
+
+Follow the steps below to test an extension running with TLS and connect to an OpenSearch node with 
+the security plugin and SSL enabled:
+
+1. Create a local distribution of [OpenSearch](https://github.com/opensearch-project/opensearch) and move the output to location you would like to install
+OpenSearch into:
+
+```
+cd opensearch
+./gradlew localDistro
+mv distribution/archives/darwin-tar/build/install/opensearch-<OPENSEARCH_VERSION>-SNAPSHOT/ ~/opensearch
+```
+
+2. Assemble the [Security plugin](https://github.com/opensearch-project/security) and move the output to the same directory
+
+```
+cd security
+./gradlew assemble
+mv build/distributions/opensearch-security-<OPENSEARCH_VERSION>.0-SNAPSHOT.zip ~/opensearch
+```
+
+3. Install the Security plugin and associated certificates created above
+
+3.1. Navigate to the root of the OpenSearch installation:
+
+```
+cd ~/opensearch/opensearch-<OPENSEARCH_VERSION>-SNAPSHOT/
+./bin/opensearch-plugin install file:$HOME/opensearch/opensearch-security-<OPENSEARCH_VERSION>.0-SNAPSHOT.zip
+```
+
+3.2 Copy the certificates generated above into `config/` directory
+
+The certificates needed are:
+
+- `os-node-01.pem`
+- `os-node-01-key.pem`
+- `root-ca.pem`
+
+3.3 Add settings in `opensearch.yml`
+
+Add the following settings in `opensearch.yml`
+
+```
+opensearch.experimental.feature.extensions.enabled: true
+plugins.security.ssl_only: true
+plugins.security.ssl.transport.pemcert_filepath: os-node-01.pem
+plugins.security.ssl.transport.pemkey_filepath: os-node-01-key.pem
+plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
+plugins.security.ssl.transport.enforce_hostname_verification: false
+plugins.security.ssl.http.enabled: true
+plugins.security.ssl.http.pemcert_filepath: os-node-01.pem
+plugins.security.ssl.http.pemkey_filepath: os-node-01-key.pem
+plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
+network.host: 0.0.0.0
+```
+
+4. Install certificates on extension
+
+Installation for the OpenSearch node is now complete, the rest of the installation is for the extension.
+
+Create a `config/` folder in the extension's home directory and install the certificates generated above.
+
+The certificates you need to add are:
+
+- `extension-01.pem`
+- `extension-01-key.pem`
+- `root-ca.pem`
+
+4.1 Add references to these certifications in extension settings file and enable SSL
+
+Add the following settings to the extension setting files. i.e. `helloworld-settings.yml`
+
+```
+ssl.transport.enabled: true
+ssl.transport.pemcert_filepath: extension-01.pem
+ssl.transport.pemkey_filepath: extension-01-key.pem
+ssl.transport.pemtrustedcas_filepath: root-ca.pem
+ssl.transport.enforce_hostname_verification: false
+path.home: <path/to/extension>
+```

src/main/java/org/opensearch/sdk/ExtensionSettings.java

@@ -15,7 +15,9 @@
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 import org.yaml.snakeyaml.Yaml;
 
@@ -30,6 +32,8 @@ public class ExtensionSettings {
     private String opensearchAddress;
     private String opensearchPort;
 
+    private Map<String, String> otherSettings;
+
     /**
      * Jackson requires a no-arg constructor.
      */
@@ -38,6 +42,33 @@ private ExtensionSettings() {
         super();
     }
 
+    /**
+     * Instantiate this class using the specified parameters.
+     *
+     * @param extensionName  The extension name. Provided to OpenSearch as a response to initialization query. Must match the defined extension name in OpenSearch.
+     * @param hostAddress  The IP Address to bind this extension to.
+     * @param hostPort  The port to bind this extension to.
+     * @param opensearchAddress  The IP Address on which OpenSearch is running.
+     * @param opensearchPort  The port on which OpenSearch is running.
+     * @param otherSettings A generic map of any settings set in the config file that are not default setting keys
+     */
+    public ExtensionSettings(
+        String extensionName,
+        String hostAddress,
+        String hostPort,
+        String opensearchAddress,
+        String opensearchPort,
+        Map<String, String> otherSettings
+    ) {
+        super();
+        this.extensionName = extensionName;
+        this.hostAddress = hostAddress;
+        this.hostPort = hostPort;
+        this.opensearchAddress = opensearchAddress;
+        this.opensearchPort = opensearchPort;
+        this.otherSettings = otherSettings;
+    }
+
     /**
      * Instantiate this class using the specified parameters.
      *
@@ -54,6 +85,7 @@ public ExtensionSettings(String extensionName, String hostAddress, String hostPo
         this.hostPort = hostPort;
         this.opensearchAddress = opensearchAddress;
         this.opensearchPort = opensearchPort;
+        this.otherSettings = Map.of();
     }
 
     public String getExtensionName() {
@@ -76,6 +108,10 @@ public String getOpensearchPort() {
         return opensearchPort;
     }
 
+    public Map<String, String> getOtherSettings() {
+        return otherSettings;
+    }
+
     @Override
     public String toString() {
         return "ExtensionSettings{extensionName="
@@ -88,6 +124,8 @@ public String toString() {
             + opensearchAddress
             + ", opensearchPort="
             + opensearchPort
+            + ", otherSettings="
+            + otherSettings
             + "}";
     }
 
@@ -109,12 +147,20 @@ public static ExtensionSettings readSettingsFromYaml(String extensionSettingsPat
             if (extensionMap == null) {
                 throw new IOException("extension.yml is empty");
             }
+            Map<String, String> otherSettings = new HashMap<>();
+            Set<String> defaultSettings = Set.of("extensionName", "hostAddress", "hostPort", "opensearchAddress", "opensearchPort");
+            for (String settingKey : extensionMap.keySet()) {
+                if (!defaultSettings.contains(settingKey)) {
+                    otherSettings.put(settingKey, extensionMap.get(settingKey).toString());
+                }
+            }
             return new ExtensionSettings(
                 extensionMap.get("extensionName").toString(),
                 extensionMap.get("hostAddress").toString(),
                 extensionMap.get("hostPort").toString(),
                 extensionMap.get("opensearchAddress").toString(),
-                extensionMap.get("opensearchPort").toString()
+                extensionMap.get("opensearchPort").toString(),
+                otherSettings
             );
         } catch (URISyntaxException e) {
             throw new IOException("Error reading from extension.yml");

src/main/java/org/opensearch/sdk/ExtensionsRunner.java

@@ -9,6 +9,10 @@
 
 package org.opensearch.sdk;
 
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Key;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.ActionType;
@@ -17,29 +21,29 @@
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.common.io.stream.NamedWriteableRegistry;
-import org.opensearch.extensions.rest.ExtensionRestRequest;
-import org.opensearch.extensions.rest.RegisterRestActionsRequest;
-import org.opensearch.extensions.settings.RegisterCustomSettingsRequest;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.discovery.InitializeExtensionRequest;
-import org.opensearch.extensions.DiscoveryExtensionNode;
 import org.opensearch.extensions.AddSettingsUpdateConsumerRequest;
-import org.opensearch.extensions.UpdateSettingsRequest;
-import org.opensearch.extensions.action.ExtensionActionRequest;
-import org.opensearch.extensions.ExtensionsManager.RequestType;
+import org.opensearch.extensions.DiscoveryExtensionNode;
 import org.opensearch.extensions.ExtensionRequest;
 import org.opensearch.extensions.ExtensionsManager;
+import org.opensearch.extensions.ExtensionsManager.RequestType;
+import org.opensearch.extensions.UpdateSettingsRequest;
+import org.opensearch.extensions.action.ExtensionActionRequest;
+import org.opensearch.extensions.rest.ExtensionRestRequest;
+import org.opensearch.extensions.rest.RegisterRestActionsRequest;
+import org.opensearch.extensions.settings.RegisterCustomSettingsRequest;
 import org.opensearch.index.IndicesModuleRequest;
 import org.opensearch.rest.RestHandler.Route;
+import org.opensearch.sdk.action.SDKActionModule;
+import org.opensearch.sdk.handlers.AcknowledgedResponseHandler;
 import org.opensearch.sdk.api.ActionExtension;
 import org.opensearch.sdk.handlers.ClusterSettingsResponseHandler;
 import org.opensearch.sdk.handlers.ClusterStateResponseHandler;
 import org.opensearch.sdk.handlers.EnvironmentSettingsResponseHandler;
 import org.opensearch.sdk.handlers.ExtensionActionRequestHandler;
-import org.opensearch.sdk.action.SDKActionModule;
-import org.opensearch.sdk.handlers.AcknowledgedResponseHandler;
 import org.opensearch.sdk.handlers.ExtensionDependencyResponseHandler;
 import org.opensearch.sdk.handlers.ExtensionsIndicesModuleNameRequestHandler;
 import org.opensearch.sdk.handlers.ExtensionsIndicesModuleRequestHandler;
@@ -56,11 +60,6 @@
 import org.opensearch.transport.TransportService;
 import org.opensearch.transport.TransportSettings;
 
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -74,6 +73,28 @@
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_CLIENT_PEMCERT_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_CLIENT_PEMKEY_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_CLIENT_PEMTRUSTEDCAS_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_ENABLED;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_ENABLED_CIPHERS;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_ENABLED_PROTOCOLS;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_EXTENDED_KEY_USAGE_ENABLED;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_KEYSTORE_ALIAS;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_KEYSTORE_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_KEYSTORE_TYPE;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_PEMCERT_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_PEMKEY_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_PEMTRUSTEDCAS_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_SERVER_PEMCERT_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_SERVER_PEMKEY_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_SERVER_PEMTRUSTEDCAS_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_TRUSTSTORE_ALIAS;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_TRUSTSTORE_FILEPATH;
+import static org.opensearch.sdk.ssl.SSLConfigConstants.SSL_TRANSPORT_TRUSTSTORE_TYPE;
+
 /**
  * The primary class to run an extension.
  * <p>
@@ -179,11 +200,37 @@ protected ExtensionsRunner(Extension extension) throws IOException {
         // These must have getters from this class to be accessible via createComponents
         // If they require later initialization, create a concrete wrapper class and update the internals
         ExtensionSettings extensionSettings = extension.getExtensionSettings();
-        this.settings = Settings.builder()
+        Settings.Builder settingsBuilder = Settings.builder()
             .put(NODE_NAME_SETTING, extensionSettings.getExtensionName())
             .put(TransportSettings.BIND_HOST.getKey(), extensionSettings.getHostAddress())
-            .put(TransportSettings.PORT.getKey(), extensionSettings.getHostPort())
-            .build();
+            .put(TransportSettings.PORT.getKey(), extensionSettings.getHostPort());
+        boolean sslEnabled = extensionSettings.getOtherSettings().containsKey(SSL_TRANSPORT_ENABLED)
+            && "true".equals(extensionSettings.getOtherSettings().get(SSL_TRANSPORT_ENABLED));
+        if (sslEnabled) {
+            addSettingsToBuilder(settingsBuilder, "path.home", extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_ENABLED, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_PEMCERT_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_PEMKEY_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_PEMTRUSTEDCAS_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_EXTENDED_KEY_USAGE_ENABLED, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_CLIENT_PEMKEY_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_CLIENT_PEMCERT_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_CLIENT_PEMTRUSTEDCAS_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_SERVER_PEMKEY_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_SERVER_PEMCERT_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_SERVER_PEMTRUSTEDCAS_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_ENABLED_PROTOCOLS, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_ENABLED_CIPHERS, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_KEYSTORE_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_KEYSTORE_ALIAS, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_KEYSTORE_TYPE, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_TRUSTSTORE_ALIAS, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_TRUSTSTORE_FILEPATH, extensionSettings);
+            addSettingsToBuilder(settingsBuilder, SSL_TRANSPORT_TRUSTSTORE_TYPE, extensionSettings);
+        }
+        this.settings = settingsBuilder.build();
 
         final List<ExecutorBuilder<?>> executorBuilders = extension.getExecutorBuilders(settings);
 
@@ -252,6 +299,12 @@ protected ExtensionsRunner(Extension extension) throws IOException {
         }
     }
 
+    private void addSettingsToBuilder(Settings.Builder settingsBuilder, String settingKey, ExtensionSettings extensionSettings) {
+        if (extensionSettings.getOtherSettings().containsKey(settingKey)) {
+            settingsBuilder.put(settingKey, extensionSettings.getOtherSettings().get(settingKey));
+        }
+    }
+
     @SuppressWarnings({ "unchecked", "rawtypes" })
     private void injectComponents(Binder b) {
         extension.createComponents(this).stream().forEach(p -> b.bind((Class) p.getClass()).toInstance(p));