Add SiLU operator

onnx · Feb 26, 2024 · 4ce1e17 · 4ce1e17
1 parent c95a59c
commit 4ce1e17
Show file tree

Hide file tree

Showing 7 changed files with 101 additions and 0 deletions.
diff --git a/docs/Changelog.md b/docs/Changelog.md
@@ -25735,6 +25735,38 @@ This version of the operator has been available since version 21 of the default
 <dd>Constrain output to int64 tensor.</dd>
 </dl>
 
+### <a name="SiLU-21"></a>**SiLU-21**</a>
+
+  Sigmoid Linear Unit (SiLU), or known as the Swish function, takes one input data (Tensor<T>) 
+  and produces one output data (Tensor<T>) of the same shape, 
+  where $silu(x) = x * sigmoid(beta * x)$.
+
+#### Version
+
+This version of the operator has been available since version 21 of the default ONNX operator set.
+
+#### Inputs
+
+<dl>
+<dt><tt>X</tt> (differentiable) : T</dt>
+<dd>Input tensor</dd>
+</dl>
+
+#### Outputs
+
+<dl>
+<dt><tt>Y</tt> (differentiable) : T</dt>
+<dd>Output tensor</dd>
+</dl>
+
+#### Type Constraints
+
+<dl>
+<dt><tt>T</tt> : tensor(float16), tensor(float), tensor(double)</dt>
+<dd>Constrain input and output types to float tensors.</dd>
+</dl>
+
+
 ### <a name="Size-21"></a>**Size-21**</a>
 
   Takes a tensor as input and outputs a int64 scalar that equals to the total number of elements of the input tensor.

diff --git a/docs/Operators.md b/docs/Operators.md
@@ -141,6 +141,7 @@ For an operator input/output's differentiability, it can be differentiable,
 |<a href="#Sign">Sign</a>|<a href="Changelog.md#Sign-13">13</a>, <a href="Changelog.md#Sign-9">9</a>|
 |<a href="#Sin">Sin</a>|<a href="Changelog.md#Sin-7">7</a>|
 |<a href="#Sinh">Sinh</a>|<a href="Changelog.md#Sinh-9">9</a>|
+|<a href="#SiLU">SiLU</a>|<a href="Changelog.md#SiLU-21">21</a>|
 |<a href="#Size">Size</a>|<a href="Changelog.md#Size-21">21</a>, <a href="Changelog.md#Size-19">19</a>, <a href="Changelog.md#Size-13">13</a>, <a href="Changelog.md#Size-1">1</a>|
 |<a href="#Slice">Slice</a>|<a href="Changelog.md#Slice-13">13</a>, <a href="Changelog.md#Slice-11">11</a>, <a href="Changelog.md#Slice-10">10</a>, <a href="Changelog.md#Slice-1">1</a>|
 |<a href="#SpaceToDepth">SpaceToDepth</a>|<a href="Changelog.md#SpaceToDepth-13">13</a>, <a href="Changelog.md#SpaceToDepth-1">1</a>|

diff --git a/docs/TestCoverage.md b/docs/TestCoverage.md
@@ -20211,6 +20211,27 @@ expect(node, inputs=[x], outputs=[y], name="test_sinh")
 </details>
 
 
+### SiLU
+There are 1 test cases, listed as following:
+<details>
+<summary>silu</summary>
+
+```python
+node = onnx.helper.make_node(
+    'SiLU',
+    inputs=['x'],
+    outputs=['y'],
+)
+x = np.array([3, 4, 5]).astype(np.float32)
+y = x * 1 / (1 + np.exp(-x))
+
+expect(node, inputs=[x], outputs=[y],
+       name='test_silu')
+```
+
+</details>
+
+
 ### Size
 There are 1 test cases, listed as following:
 <details>

diff --git a/onnx/backend/test/case/node/silu.py b/onnx/backend/test/case/node/silu.py
@@ -0,0 +1,23 @@
+# Copyright (c) ONNX Project Contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import numpy as np
+
+import onnx
+from onnx.backend.test.case.base import Base
+from onnx.backend.test.case.node import expect
+
+
+class SiLU(Base):
+    @staticmethod
+    def export() -> None:
+        node = onnx.helper.make_node(
+            "SiLU",
+            inputs=["x"],
+            outputs=["y"],
+        )
+
+        x = np.array([3, 4, 5]).astype(np.float32)
+        y = x * 1 / (1 + np.exp(-x))
+        expect(node, inputs=[x], outputs=[y], name="test_silu")
diff --git a/onnx/defs/math/defs.cc b/onnx/defs/math/defs.cc
@@ -644,6 +644,25 @@ ONNX_OPERATOR_SET_SCHEMA(
         .SetContextDependentFunctionBodyBuilder(BuildContextDependentFunctionBodyGelu)
         .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput));
 
+static const char* SiLU_ver21_doc = R"DOC(
+Sigmoid Linear Unit (SiLU), or known as the Swish function, takes one input data (Tensor<T>) 
+and produces one output data (Tensor<T>) of the same shape, 
+where $silu(x) = x * sigmoid(beta * x)$.
+)DOC";
+
+ONNX_OPERATOR_SET_SCHEMA(
+    SiLU,
+    21,
+    OpSchema()
+        .SetDoc(SiLU_ver21_doc)
+        .Input(0, "X", "Input tensor", "T", OpSchema::Single, true, 1, OpSchema::Differentiable)
+        .Output(0, "Y", "Output tensor", "T", OpSchema::Single, true, 1, OpSchema::Differentiable)
+        .TypeConstraint(
+            "T",
+            {"tensor(float16)", "tensor(float)", "tensor(double)", "tensor(bfloat16)"},
+            "Constrain input and output types to float tensors.")
+        .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput));
+
 static const char* Exp_ver13_doc = R"DOC(
 Calculates the exponential of the given input tensor, element-wise.
 )DOC";

diff --git a/onnx/defs/operator_sets.h b/onnx/defs/operator_sets.h
@@ -1157,6 +1157,7 @@ class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 21, Size);
 class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 21, Squeeze);
 class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 21, Transpose);
 class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 21, Unsqueeze);
+class ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 21, SiLU);
 
 // Iterate over schema from ai.onnx version 21
 class OpSet_Onnx_ver21 {
@@ -1182,6 +1183,7 @@ class OpSet_Onnx_ver21 {
     fn(GetOpSchema<ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 21, Squeeze)>());
     fn(GetOpSchema<ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 21, Transpose)>());
     fn(GetOpSchema<ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 21, Unsqueeze)>());
+    fn(GetOpSchema<ONNX_OPERATOR_SET_SCHEMA_CLASS_NAME(Onnx, 21, SiLU)>());
   }
 };
 

diff --git a/onnx/test/version_converter/automatic_upgrade_test.py b/onnx/test/version_converter/automatic_upgrade_test.py
@@ -1195,6 +1195,9 @@ def test_Sigmoid(self) -> None:
 
     def test_Sign(self) -> None:
         self._test_op_upgrade("Sign", 9)
+
+    def test_SiLU(self) -> None:
+        self._test_op_upgrade("SiLU", 21)
 
     def test_Sinh(self) -> None:
         self._test_op_upgrade("Sinh", 9)