From b2764cf337a5ec5eeebb79888cec97062f6e19cc Mon Sep 17 00:00:00 2001
From: Leon Chen <leon@md.ai>
Date: Mon, 10 Oct 2016 22:24:32 -0400
Subject: [PATCH] fix softmax activation for large values

---
 notebooks/activations.ipynb          | 47 ++++++++++++++++++++++++++--
 src/activations.js                   |  4 +++
 test/activations/activations.js      | 16 ++++++++++
 test/activations/data_activations.js |  4 +++
 4 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/notebooks/activations.ipynb b/notebooks/activations.ipynb
index 749cecf..e1d6ccc 100644
--- a/notebooks/activations.ipynb
+++ b/notebooks/activations.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {
     "collapsed": false
    },
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 3,
    "metadata": {
     "collapsed": true
    },
@@ -120,6 +120,46 @@
     "print('out:', data_out)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**[activations.softmax.2] 1D**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "in: [0, 0.2, 0.5, -0.1, 1, 90]\n",
+      "in shape: (6,)\n",
+      "out shape: (6,)\n",
+      "out: [0.0, 0.0, 0.0, 0.0, 0.0, 1.0]\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_in = [0, 0.2, 0.5, -0.1, 1, 90]\n",
+    "data_in_shape = (6,)\n",
+    "print('in:', data_in)\n",
+    "print('in shape:', data_in_shape)\n",
+    "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n",
+    "\n",
+    "result = activations.softmax(K.variable(np.array([arr_in])))\n",
+    "\n",
+    "arr_out = K.eval(result)[0]\n",
+    "print('out shape:', arr_out.shape)\n",
+    "data_out = format_decimal(arr_out.ravel().tolist())\n",
+    "print('out:', data_out)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -1100,8 +1140,9 @@
   }
  ],
  "metadata": {
+  "anaconda-cloud": {},
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python [default]",
    "language": "python",
    "name": "python3"
   },
diff --git a/src/activations.js b/src/activations.js
index e327036..73dcf19 100644
--- a/src/activations.js
+++ b/src/activations.js
@@ -9,11 +9,15 @@ import Tensor from './Tensor'
  */
 export function softmax (x) {
   if (x.tensor.shape.length === 1) {
+    const maxval = ops.sup(x.tensor)
+    ops.subseq(x.tensor, maxval)
     ops.expeq(x.tensor)
     const sum = ops.sum(x.tensor)
     ops.divseq(x.tensor, sum)
   } else if (x.tensor.shape.length === 2) {
     for (let i = 0; i < x.tensor.shape[0]; i++) {
+      const maxval = ops.sup(x.tensor.pick(i, null))
+      ops.subseq(x.tensor.pick(i, null), maxval)
       ops.expeq(x.tensor.pick(i, null))
       const sum = ops.sum(x.tensor.pick(i, null))
       ops.divseq(x.tensor.pick(i, null), sum)
diff --git a/test/activations/activations.js b/test/activations/activations.js
index 70f6c1e..69f50e3 100644
--- a/test/activations/activations.js
+++ b/test/activations/activations.js
@@ -52,6 +52,22 @@ describe('activations', function () {
       assert.deepEqual(t.tensor.shape, shapeExpected)
       assert.isTrue(approxEquals(t.tensor, dataExpected))
     })
+
+    it('[activations.softmax.2] should work for very large values', function () {
+      const key = 'activations.softmax.2'
+      console.log(`\n%c[${key}] 1D, large values`, styles.h3)
+      let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
+      console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+      const startTime = performance.now()
+      activations.softmax(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
+      const shapeExpected = TEST_DATA[key].expected.shape
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
   })
 
   /*********************************************************
diff --git a/test/activations/data_activations.js b/test/activations/data_activations.js
index ed7a75b..8cf498d 100644
--- a/test/activations/data_activations.js
+++ b/test/activations/data_activations.js
@@ -12,6 +12,10 @@
       input: { data: [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], shape: [2, 6] },
       expected: { data: [0.067194, 0.082071, 0.110784, 0.0608, 0.182652, 0.4965, 0.107768, 0.149902, 0.11105, 0.247147, 0.082268, 0.301865], shape: [2, 6] }
     },
+    'activations.softmax.2': {
+      input: { data: [0, 0.2, 0.5, -0.1, 1, 90], shape: [6] },
+      expected: { data: [0.0, 0.0, 0.0, 0.0, 0.0, 1.0], shape: [6] }
+    },
     'activations.softplus.0': {
       input: { data: [0, 0.2, 0.5, -0.1, 1, 2], shape: [6] },
       expected: { data: [0.693147, 0.798139, 0.974077, 0.644397, 1.313262, 2.126928], shape: [6] }