Add Swish operator

docs/Changelog.md

@@ -28712,4 +28712,3 @@ This version of the operator has been available since version 1 of the 'ai.onnx.
 <dt><tt>T3</tt> : tensor(float), tensor(double)</dt>
 <dd>Constrain input types to float tensors.</dd>
 </dl>
-

docs/Operators.md

@@ -36187,4 +36187,3 @@ expect(
 
 </details>
 
-

onnx/backend/test/case/node/swish.py

@@ -0,0 +1,29 @@
+# Copyright (c) ONNX Project Contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import numpy as np
+
+import onnx
+from onnx.backend.test.case.base import Base
+from onnx.backend.test.case.node import expect
+
+
+def swish(x: np.ndarray, alpha: np.float16) -> np.ndarray:
+    return x * 1 / (1 + np.exp(np.negative(x * alpha)))
+
+
+class Swish(Base):
+    @staticmethod
+    def export() -> None:
+        node = onnx.helper.make_node(
+            "Swish",
+            inputs=["x"],
+            outputs=["y"],
+        )
+
+        x = np.array([3, 4, 5]).astype(np.float32)
+        y = swish(x, alpha=1.0)
+
+        expect(node, inputs=[x], outputs=[y], name="test_swish")

onnx/backend/test/data/node/test_dequantizelinear_int4/test_data_set_0/input_2.pb

@@ -1 +1,2 @@
-
*

Bx_zero_point
+
*

B
+zero_point

onnx/backend/test/data/node/test_dequantizelinear_uint4/test_data_set_0/input_2.pb

@@ -1 +1,2 @@
-
*

Bx_zero_point
+
*

B
+zero_point

onnx/backend/test/data/node/test_quantizelinear_int4/test_data_set_0/input_2.pb

@@ -1 +1,2 @@
-*
By_zero_point
+*
B
+zero_point

onnx/backend/test/data/node/test_quantizelinear_uint4/test_data_set_0/input_2.pb

@@ -1 +1,2 @@
-*
By_zero_point
+*
B
+zero_point

onnx/defs/math/defs.cc

@@ -621,6 +621,35 @@ ONNX_OPERATOR_SET_SCHEMA(
         .SetContextDependentFunctionBodyBuilder(BuildContextDependentFunctionBodyGelu)
         .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput));
 
+static const char* Swish_ver22_doc = R"DOC(
+Swish function takes one input data (Tensor<T>) and produces one output data (Tensor<T>) of the same shape,
+where $Swish(x) = x * sigmoid(beta * x)$.
+)DOC";
+
+ONNX_OPERATOR_SET_SCHEMA(
+    Swish,
+    22,
+    OpSchema()
+        .SetDoc(Swish_ver22_doc)
+        .Input(0, "X", "Input tensor", "T", OpSchema::Single, true, 1, OpSchema::Differentiable)
+        .Output(0, "Y", "Output tensor", "T", OpSchema::Single, true, 1, OpSchema::Differentiable)
+        .TypeConstraint(
+            "T",
+            {"tensor(float16)", "tensor(float)", "tensor(bfloat16)"},
+            "Constrain input and output types to float tensors.")
+        .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput)
+        .FunctionBody(
+            R"ONNX(
+          {
+            Alpha =  Constant <value_float: float = @alpha>()
+            AlphaCast = CastLike (Alpha, X)
+            AlphaMulX = Mul (AlphaCast, X)
+            SigmoidAlphaMulX = Sigmoid(AlphaMulX)
+            Y = Mul (X, SigmoidAlphaMulX)
+          }
+        )ONNX",
+            22));
+
 static const char* Exp_ver13_doc = R"DOC(
 Calculates the exponential of the given input tensor, element-wise.
 )DOC";

onnx/defs/operator_sets.h

@@ -1234,6 +1234,7 @@ class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 22, RNN);
 class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 22, GRU);
 class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 22, LSTM);
 class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 22, GridSample);
+class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 22, Swish);
 
 // Iterate over schema from ai.onnx version 22
 class OpSet_Onnx_ver22 {
@@ -1287,6 +1288,7 @@ class OpSet_Onnx_ver22 {
     fn(GetOpSchema<ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 22, GRU)>());
     fn(GetOpSchema<ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 22, LSTM)>());
     fn(GetOpSchema<ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 22, GridSample)>());
+    fn(GetOpSchema<ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 22, Swish)>());
   }
 };
 

onnx/test/version_converter/automatic_upgrade_test.py

@@ -1301,6 +1301,9 @@ def test_Sum(self) -> None:
             attrs={"consumed_inputs": [0]},
         )
 
+    def test_Swish(self) -> None:
+        self._test_op_upgrade("Swish", 22)
+
     def test_Tanh(self) -> None:
         self._test_op_upgrade("Tanh", 1, attrs={"consumed_inputs": [0]})