diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py
index a9d709d..f83f044 100644
--- a/keras_contrib/layers/advanced_activations.py
+++ b/keras_contrib/layers/advanced_activations.py
@@ -66,14 +66,14 @@ class PELU(Layer):
 
     def call(self, x, mask=None):
         if K.backend() == 'theano':
-            pos = K.relu(x) * (K.pattern_broadcast(self.alphas, self.param_broadcast) /
+            pos = x * (K.pattern_broadcast(self.alphas, self.param_broadcast) /
                                K.pattern_broadcast(self.betas, self.param_broadcast))
             neg = (K.pattern_broadcast(self.alphas, self.param_broadcast) *
                    (K.exp(x / K.pattern_broadcast(self.betas, self.param_broadcast)) - 1))
         else:
             pos = K.relu(x) * (self.alphas / self.betas)
             neg = self.alphas * (K.exp(x / self.betas) - 1)
-        return pos + neg
+        return K.switch(x < 0., neg, pos)
 
     def get_config(self):
         config = {'alphas_init': self.alphas_init.__name__,