From c38dd04e98d55f4ecdb63ea444d3d43b6bdbbce7 Mon Sep 17 00:00:00 2001 From: Dmytro Doroshenko Date: Thu, 8 Oct 2020 12:51:12 +0300 Subject: [PATCH 1/9] small docs fix --- catalyst/contrib/nn/modules/arcface.py | 3 ++- catalyst/contrib/nn/modules/cosface.py | 3 ++- catalyst/contrib/nn/modules/softmax.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/catalyst/contrib/nn/modules/arcface.py b/catalyst/contrib/nn/modules/arcface.py index e2d2756b1a..a6a1bdb75e 100644 --- a/catalyst/contrib/nn/modules/arcface.py +++ b/catalyst/contrib/nn/modules/arcface.py @@ -89,7 +89,8 @@ def forward(self, input, target): Returns: tensor (logits) with shapes ``BxC`` - where ``C`` is a number of classes. + where ``C`` is a number of classes + (out_features). """ cos_theta = F.linear(F.normalize(input), F.normalize(self.weight)) theta = torch.acos( diff --git a/catalyst/contrib/nn/modules/cosface.py b/catalyst/contrib/nn/modules/cosface.py index 7215e12615..83ef6a21cb 100644 --- a/catalyst/contrib/nn/modules/cosface.py +++ b/catalyst/contrib/nn/modules/cosface.py @@ -72,7 +72,8 @@ def forward(self, input, target): Returns: tensor (logits) with shapes ``BxC`` - where ``C`` is a number of classes. + where ``C`` is a number of classes + (out_features). """ cosine = F.linear(F.normalize(input), F.normalize(self.weight)) phi = cosine - self.m diff --git a/catalyst/contrib/nn/modules/softmax.py b/catalyst/contrib/nn/modules/softmax.py index 89d04d9162..ab14737870 100644 --- a/catalyst/contrib/nn/modules/softmax.py +++ b/catalyst/contrib/nn/modules/softmax.py @@ -59,6 +59,7 @@ def forward(self, input): Returns: tensor (logits) with shapes ``BxC`` - where ``C`` is a number of classes. + where ``C`` is a number of classes + (out_features). """ return F.linear(input, self.weight, self.bias) From 964fefa460c3325a2babd44d35931b6fae3a138e Mon Sep 17 00:00:00 2001 From: Dmytro Doroshenko Date: Thu, 8 Oct 2020 13:00:09 +0300 Subject: [PATCH 2/9] arcmargin & typings --- catalyst/contrib/nn/modules/__init__.py | 1 + catalyst/contrib/nn/modules/arcface.py | 8 +++- catalyst/contrib/nn/modules/arcmargin.py | 49 ++++++++++++++++++++++++ catalyst/contrib/nn/modules/cosface.py | 4 +- catalyst/contrib/nn/modules/softmax.py | 2 +- docs/api/contrib.rst | 7 ++++ 6 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 catalyst/contrib/nn/modules/arcmargin.py diff --git a/catalyst/contrib/nn/modules/__init__.py b/catalyst/contrib/nn/modules/__init__.py index 8ec4d226c7..eef1ef010b 100644 --- a/catalyst/contrib/nn/modules/__init__.py +++ b/catalyst/contrib/nn/modules/__init__.py @@ -35,3 +35,4 @@ from catalyst.contrib.nn.modules.softmax import SoftMax from catalyst.contrib.nn.modules.arcface import ArcFace, SubCenterArcFace from catalyst.contrib.nn.modules.cosface import CosFace +from catalyst.contrib.nn.modules.arcmargin import ArcMarginProduct diff --git a/catalyst/contrib/nn/modules/arcface.py b/catalyst/contrib/nn/modules/arcface.py index a6a1bdb75e..7d0f09b2d5 100644 --- a/catalyst/contrib/nn/modules/arcface.py +++ b/catalyst/contrib/nn/modules/arcface.py @@ -76,7 +76,9 @@ def __repr__(self) -> str: + ")" ) - def forward(self, input, target): + def forward( + self, input: torch.FloatTensor, target: torch.LongTensor + ) -> torch.FloatTensor: """ Args: input: input features, @@ -190,7 +192,9 @@ def __repr__(self) -> str: + ")" ) - def forward(self, input, label): + def forward( + self, input: torch.FloatTensor, label: torch.LongTensor + ) -> torch.FloatTensor: """ Args: input: input features, diff --git a/catalyst/contrib/nn/modules/arcmargin.py b/catalyst/contrib/nn/modules/arcmargin.py new file mode 100644 index 0000000000..1d866c2bc5 --- /dev/null +++ b/catalyst/contrib/nn/modules/arcmargin.py @@ -0,0 +1,49 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class ArcMarginProduct(nn.Module): + """Implementation of Arc Margin Product. + + Args: + in_features: size of each input sample. + out_features: size of each output sample. + + Shape: + - Input: :math:`(batch, H_{in})` where + :math:`H_{in} = in\_features`. + - Output: :math:`(batch, H_{out})` where + :math:`H_{out} = out\_features`. + + Example: + >>> layer = ArcMarginProduct(5, 10) + >>> loss_fn = nn.CrosEntropyLoss() + >>> embedding = torch.randn(3, 5, requires_grad=True) + >>> target = torch.empty(3, dtype=torch.long).random_(10) + >>> output = layer(embedding) + >>> loss = loss_fn(output, target) + >>> loss.backward() + + """ + + def __init__(self, in_features: int, out_features: int): # noqa: D107 + super(ArcMarginProduct, self).__init__() + self.weight = nn.Parameter(torch.Tensor(out_features, in_features)) + nn.init.xavier_uniform_(self.weight) + + def forward(self, input: torch.FloatTensor) -> torch.FloatTensor: + """ + Args: + input: input features, + expected shapes ``BxF`` where ``B`` + is batch dimension and ``F`` is an + input feature dimension. + + Returns: + tensor (logits) with shapes ``BxC`` + where ``C`` is a number of classes + (out_features). + """ + cosine = F.linear(F.normalize(input), F.normalize(self.weight)) + return cosine diff --git a/catalyst/contrib/nn/modules/cosface.py b/catalyst/contrib/nn/modules/cosface.py index 83ef6a21cb..5e6ba5a1b2 100644 --- a/catalyst/contrib/nn/modules/cosface.py +++ b/catalyst/contrib/nn/modules/cosface.py @@ -59,7 +59,9 @@ def __repr__(self) -> str: self.in_features, self.out_features, self.s, self.m ) - def forward(self, input, target): + def forward( + self, input: torch.FloatTensor, target: torch.LongTensor + ) -> torch.LongTensor: """ Args: input: input features, diff --git a/catalyst/contrib/nn/modules/softmax.py b/catalyst/contrib/nn/modules/softmax.py index ab14737870..fb595ffd03 100644 --- a/catalyst/contrib/nn/modules/softmax.py +++ b/catalyst/contrib/nn/modules/softmax.py @@ -49,7 +49,7 @@ def __repr__(self) -> str: self.in_features, self.out_features ) - def forward(self, input): + def forward(self, input: torch.FloatTensor) -> torch.FloatTensor: """ Args: input: input features, diff --git a/docs/api/contrib.rst b/docs/api/contrib.rst index d48bbbb9a7..ddb21120dc 100644 --- a/docs/api/contrib.rst +++ b/docs/api/contrib.rst @@ -257,6 +257,13 @@ ArcFace and SubCenterArcFace :undoc-members: :show-inheritance: +Arc Margin Product +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +.. automodule:: catalyst.contrib.nn.modules.arcmargin + :members: + :undoc-members: + :show-inheritance: + Common modules """""""""""""""""""""""""""""""""""""""""" .. automodule:: catalyst.contrib.nn.modules.common From 2e9d201448e1080b850a43389b7b4759c21dbe7f Mon Sep 17 00:00:00 2001 From: Dmytro Doroshenko Date: Thu, 8 Oct 2020 13:03:24 +0300 Subject: [PATCH 3/9] arc margin product --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38dc6b7a63..8977c48ce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - docs for MetricCallbacks ([#947](https://github.com/catalyst-team/catalyst/pull/947)) - SoftMax, CosFace, ArcFace layers to contrib ([#939](https://github.com/catalyst-team/catalyst/pull/939)) +- ArcMargin layer to contrib ([#957](https://github.com/catalyst-team/catalyst/pull/957)) ### Changed From 7f157aa6c89b0409e5c95da04431d6cdacf8ae37 Mon Sep 17 00:00:00 2001 From: Dmytro Doroshenko Date: Fri, 9 Oct 2020 16:51:03 +0300 Subject: [PATCH 4/9] fixed typings --- catalyst/contrib/nn/modules/arcface.py | 12 ++++++------ catalyst/contrib/nn/modules/arcmargin.py | 2 +- catalyst/contrib/nn/modules/cosface.py | 8 +++++--- catalyst/contrib/nn/modules/softmax.py | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/catalyst/contrib/nn/modules/arcface.py b/catalyst/contrib/nn/modules/arcface.py index 7d0f09b2d5..72144a6cae 100644 --- a/catalyst/contrib/nn/modules/arcface.py +++ b/catalyst/contrib/nn/modules/arcface.py @@ -77,8 +77,8 @@ def __repr__(self) -> str: ) def forward( - self, input: torch.FloatTensor, target: torch.LongTensor - ) -> torch.FloatTensor: + self, input: torch.Tensor, target: torch.LongTensor + ) -> torch.Tensor: """ Args: input: input features, @@ -99,7 +99,7 @@ def forward( torch.clamp(cos_theta, -1.0 + self.eps, 1.0 - self.eps) ) - one_hot = torch.zeros_like(cos_theta, device=input.device) + one_hot = torch.zeros_like(cos_theta) one_hot.scatter_(1, target.view(-1, 1).long(), 1) mask = torch.where( @@ -193,8 +193,8 @@ def __repr__(self) -> str: ) def forward( - self, input: torch.FloatTensor, label: torch.LongTensor - ) -> torch.FloatTensor: + self, input: torch.Tensor, label: torch.LongTensor + ) -> torch.Tensor: """ Args: input: input features, @@ -222,7 +222,7 @@ def forward( torch.clamp(cos_theta, -1.0 + self.eps, 1.0 - self.eps) ) - one_hot = torch.zeros(cos_theta.size()).to(input.device) + one_hot = torch.zeros_like(cos_theta) one_hot.scatter_(1, label.view(-1, 1).long(), 1) selected = torch.where( diff --git a/catalyst/contrib/nn/modules/arcmargin.py b/catalyst/contrib/nn/modules/arcmargin.py index 1d866c2bc5..0707e96278 100644 --- a/catalyst/contrib/nn/modules/arcmargin.py +++ b/catalyst/contrib/nn/modules/arcmargin.py @@ -32,7 +32,7 @@ def __init__(self, in_features: int, out_features: int): # noqa: D107 self.weight = nn.Parameter(torch.Tensor(out_features, in_features)) nn.init.xavier_uniform_(self.weight) - def forward(self, input: torch.FloatTensor) -> torch.FloatTensor: + def forward(self, input: torch.Tensor) -> torch.Tensor: """ Args: input: input features, diff --git a/catalyst/contrib/nn/modules/cosface.py b/catalyst/contrib/nn/modules/cosface.py index 5e6ba5a1b2..655352919e 100644 --- a/catalyst/contrib/nn/modules/cosface.py +++ b/catalyst/contrib/nn/modules/cosface.py @@ -60,8 +60,8 @@ def __repr__(self) -> str: ) def forward( - self, input: torch.FloatTensor, target: torch.LongTensor - ) -> torch.LongTensor: + self, input: torch.Tensor, target: torch.LongTensor + ) -> torch.Tensor: """ Args: input: input features, @@ -79,8 +79,10 @@ def forward( """ cosine = F.linear(F.normalize(input), F.normalize(self.weight)) phi = cosine - self.m - one_hot = torch.zeros(cosine.size()).to(input.device) + + one_hot = torch.zeros_like(cosine) one_hot.scatter_(1, target.view(-1, 1).long(), 1) + logits = (one_hot * phi) + ((1.0 - one_hot) * cosine) logits *= self.s diff --git a/catalyst/contrib/nn/modules/softmax.py b/catalyst/contrib/nn/modules/softmax.py index fb595ffd03..63061a8815 100644 --- a/catalyst/contrib/nn/modules/softmax.py +++ b/catalyst/contrib/nn/modules/softmax.py @@ -49,7 +49,7 @@ def __repr__(self) -> str: self.in_features, self.out_features ) - def forward(self, input: torch.FloatTensor) -> torch.FloatTensor: + def forward(self, input: torch.Tensor) -> torch.Tensor: """ Args: input: input features, From 227ac3a862dfa3fb0b03453f2fdeed5bc74b6931 Mon Sep 17 00:00:00 2001 From: Dmytro Doroshenko Date: Fri, 9 Oct 2020 16:54:02 +0300 Subject: [PATCH 5/9] updated repr methods --- catalyst/contrib/nn/modules/arcface.py | 3 ++- catalyst/contrib/nn/modules/arcmargin.py | 9 +++++++++ catalyst/contrib/nn/modules/cosface.py | 3 ++- catalyst/contrib/nn/modules/softmax.py | 3 ++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/catalyst/contrib/nn/modules/arcface.py b/catalyst/contrib/nn/modules/arcface.py index 72144a6cae..c97c0db7b1 100644 --- a/catalyst/contrib/nn/modules/arcface.py +++ b/catalyst/contrib/nn/modules/arcface.py @@ -62,7 +62,7 @@ def __init__( # noqa: D107 def __repr__(self) -> str: """Object representation.""" - return ( + rep = ( "ArcFace(" + ",".join( [ @@ -75,6 +75,7 @@ def __repr__(self) -> str: ) + ")" ) + return rep def forward( self, input: torch.Tensor, target: torch.LongTensor diff --git a/catalyst/contrib/nn/modules/arcmargin.py b/catalyst/contrib/nn/modules/arcmargin.py index 0707e96278..d730b95dcb 100644 --- a/catalyst/contrib/nn/modules/arcmargin.py +++ b/catalyst/contrib/nn/modules/arcmargin.py @@ -29,9 +29,18 @@ class ArcMarginProduct(nn.Module): def __init__(self, in_features: int, out_features: int): # noqa: D107 super(ArcMarginProduct, self).__init__() + self.in_features = in_features + self.out_features = out_features self.weight = nn.Parameter(torch.Tensor(out_features, in_features)) nn.init.xavier_uniform_(self.weight) + def __repr__(self) -> str: + """Object representation.""" + rep = "ArcMarginProduct(in_features={},out_features={})".format( + self.in_features, self.out_features + ) + return rep + def forward(self, input: torch.Tensor) -> torch.Tensor: """ Args: diff --git a/catalyst/contrib/nn/modules/cosface.py b/catalyst/contrib/nn/modules/cosface.py index 655352919e..2ae1b77a0d 100644 --- a/catalyst/contrib/nn/modules/cosface.py +++ b/catalyst/contrib/nn/modules/cosface.py @@ -55,9 +55,10 @@ def __init__( # noqa: D107 def __repr__(self) -> str: """Object representation.""" - return "CosFace(in_features={},out_features={},s={},m={})".format( + rep = "CosFace(in_features={},out_features={},s={},m={})".format( self.in_features, self.out_features, self.s, self.m ) + return rep def forward( self, input: torch.Tensor, target: torch.LongTensor diff --git a/catalyst/contrib/nn/modules/softmax.py b/catalyst/contrib/nn/modules/softmax.py index 63061a8815..e1761d1ef7 100644 --- a/catalyst/contrib/nn/modules/softmax.py +++ b/catalyst/contrib/nn/modules/softmax.py @@ -45,9 +45,10 @@ def __init__(self, in_features: int, num_classes: int): # noqa: D107 def __repr__(self) -> str: """Object representation.""" - return "SoftMax(in_features={},out_features={})".format( + rep = "SoftMax(in_features={},out_features={})".format( self.in_features, self.out_features ) + return rep def forward(self, input: torch.Tensor) -> torch.Tensor: """ From c73a4cb2c51e3c1b2f607ea542be868e67eabb92 Mon Sep 17 00:00:00 2001 From: Dmytro Doroshenko Date: Fri, 9 Oct 2020 16:54:59 +0300 Subject: [PATCH 6/9] updated repr --- catalyst/contrib/nn/modules/arcface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/catalyst/contrib/nn/modules/arcface.py b/catalyst/contrib/nn/modules/arcface.py index c97c0db7b1..3e76d2f5ae 100644 --- a/catalyst/contrib/nn/modules/arcface.py +++ b/catalyst/contrib/nn/modules/arcface.py @@ -178,7 +178,7 @@ def __init__( # noqa: D107 def __repr__(self) -> str: """Object representation.""" - return ( + rep = ( "SubCenterArcFace(" + ",".join( [ @@ -192,6 +192,7 @@ def __repr__(self) -> str: ) + ")" ) + return rep def forward( self, input: torch.Tensor, label: torch.LongTensor From 427ca7a28edd6d6065c4335dada0d3e9a2707e40 Mon Sep 17 00:00:00 2001 From: Dmytro Doroshenko Date: Fri, 9 Oct 2020 17:02:28 +0300 Subject: [PATCH 7/9] removed join --- catalyst/contrib/nn/modules/arcface.py | 34 ++++++++++---------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/catalyst/contrib/nn/modules/arcface.py b/catalyst/contrib/nn/modules/arcface.py index 3e76d2f5ae..9ecfab120e 100644 --- a/catalyst/contrib/nn/modules/arcface.py +++ b/catalyst/contrib/nn/modules/arcface.py @@ -64,16 +64,12 @@ def __repr__(self) -> str: """Object representation.""" rep = ( "ArcFace(" - + ",".join( - [ - f"in_features={self.in_features}", - f"out_features={self.out_features}", - f"s={self.s}", - f"m={self.m}", - f"eps={self.eps}", - ] - ) - + ")" + f"in_features={self.in_features}," + f"out_features={self.out_features}," + f"s={self.s}," + f"m={self.m}," + f"eps={self.eps}" + ")" ) return rep @@ -180,17 +176,13 @@ def __repr__(self) -> str: """Object representation.""" rep = ( "SubCenterArcFace(" - + ",".join( - [ - f"in_features={self.in_features}", - f"out_features={self.out_features}", - f"s={self.s}", - f"m={self.m}", - f"k={self.k}", - f"eps={self.eps}", - ] - ) - + ")" + f"in_features={self.in_features}," + f"out_features={self.out_features}," + f"s={self.s}," + f"m={self.m}," + f"k={self.k}," + f"eps={self.eps}" + ")" ) return rep From 8d76b14a9712040e1a30af3f99b5c2c69f5737e9 Mon Sep 17 00:00:00 2001 From: Dmytro Doroshenko Date: Sat, 10 Oct 2020 13:40:07 +0300 Subject: [PATCH 8/9] f-strings in repr & lovercased variable --- catalyst/contrib/nn/modules/arcmargin.py | 7 +++++-- catalyst/contrib/nn/modules/cosface.py | 13 +++++++++---- catalyst/contrib/nn/modules/softmax.py | 7 +++++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/catalyst/contrib/nn/modules/arcmargin.py b/catalyst/contrib/nn/modules/arcmargin.py index d730b95dcb..b9c394fe8c 100644 --- a/catalyst/contrib/nn/modules/arcmargin.py +++ b/catalyst/contrib/nn/modules/arcmargin.py @@ -36,8 +36,11 @@ def __init__(self, in_features: int, out_features: int): # noqa: D107 def __repr__(self) -> str: """Object representation.""" - rep = "ArcMarginProduct(in_features={},out_features={})".format( - self.in_features, self.out_features + rep = ( + "ArcMarginProduct(" + f"in_features={self.in_features}," + f"out_features={self.out_features}" + ")" ) return rep diff --git a/catalyst/contrib/nn/modules/cosface.py b/catalyst/contrib/nn/modules/cosface.py index 55b1f5f475..77ff1ea67e 100644 --- a/catalyst/contrib/nn/modules/cosface.py +++ b/catalyst/contrib/nn/modules/cosface.py @@ -57,8 +57,13 @@ def __init__( # noqa: D107 def __repr__(self) -> str: """Object representation.""" - rep = "CosFace(in_features={},out_features={},s={},m={})".format( - self.in_features, self.out_features, self.s, self.m + rep = ( + "CosFace(" + f"in_features={self.in_features}," + f"out_features={self.out_features}," + f"s={self.s}," + f"m={self.m}" + ")" ) return rep @@ -185,7 +190,7 @@ def forward( if self.train: with torch.no_grad(): - B_avg = ( + b_avg = ( torch.where( one_hot < 1, torch.exp(self.s * cos_theta), @@ -198,7 +203,7 @@ def forward( theta_median = torch.min( torch.full_like(theta_median, math.pi / 4), theta_median ) - self.s = (torch.log(B_avg) / torch.cos(theta_median)).item() + self.s = (torch.log(b_avg) / torch.cos(theta_median)).item() logits = self.s * cos_theta return logits diff --git a/catalyst/contrib/nn/modules/softmax.py b/catalyst/contrib/nn/modules/softmax.py index e1761d1ef7..3051027628 100644 --- a/catalyst/contrib/nn/modules/softmax.py +++ b/catalyst/contrib/nn/modules/softmax.py @@ -45,8 +45,11 @@ def __init__(self, in_features: int, num_classes: int): # noqa: D107 def __repr__(self) -> str: """Object representation.""" - rep = "SoftMax(in_features={},out_features={})".format( - self.in_features, self.out_features + rep = ( + "SoftMax(" + f"in_features={self.in_features}," + f"out_features={self.out_features}" + ")" ) return rep From 7810d11b9c7d3bbfa08498471c335a1df7eee3a3 Mon Sep 17 00:00:00 2001 From: Dmytro Doroshenko Date: Sat, 10 Oct 2020 14:41:43 +0300 Subject: [PATCH 9/9] fixed codestyle --- catalyst/contrib/nn/modules/cosface.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/catalyst/contrib/nn/modules/cosface.py b/catalyst/contrib/nn/modules/cosface.py index 77ff1ea67e..a6cb4e4827 100644 --- a/catalyst/contrib/nn/modules/cosface.py +++ b/catalyst/contrib/nn/modules/cosface.py @@ -99,10 +99,9 @@ def forward( class AdaCos(nn.Module): """Implementation of - `AdaCos: Adaptively Scaling Cosine Logits for \ - Effectively Learning Deep Face Representations`_. + `AdaCos\: Adaptively Scaling Cosine Logits for Effectively Learning Deep Face Representations`_. - .. _AdaCos\: Adaptively Scaling Cosine Logits for \ + .. _AdaCos\: Adaptively Scaling Cosine Logits for\ Effectively Learning Deep Face Representations: https://arxiv.org/abs/1905.00292 @@ -130,7 +129,7 @@ class AdaCos(nn.Module): >>> loss = loss_fn(output, target) >>> loss.backward() - """ + """ # noqa: E501,W505 def __init__( # noqa: D107 self,