entropy-dice.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "c313f401-bf96-46d3-91bc-175ec823ac59",
   "metadata": {},
   "outputs": [],
   "source": [
    "using Ipopt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "1b56f01b-429d-4e2b-8264-868ce7e3cc03",
   "metadata": {},
   "outputs": [],
   "source": [
    "using JuMP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "f65514e1-a9c4-44cc-ac48-a820fbce97c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "A JuMP Model\n",
       "Feasibility problem with:\n",
       "Variables: 0\n",
       "Model mode: AUTOMATIC\n",
       "CachingOptimizer state: EMPTY_OPTIMIZER\n",
       "Solver name: Ipopt"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = Model(Ipopt.Optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "ce681950-0ea9-4a60-aa91-cce9998ac72b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6-element Vector{VariableRef}:\n",
       " p[1]\n",
       " p[2]\n",
       " p[3]\n",
       " p[4]\n",
       " p[5]\n",
       " p[6]"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@variable(m, p[1:6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "9274f3a2-144f-4c35-9440-cf439847f839",
   "metadata": {},
   "outputs": [],
   "source": [
    "function mylog(x)\n",
    "    if x == 0.0\n",
    "        return 0.0\n",
    "    else\n",
    "        return log(x)\n",
    "    end\n",
    "end\n",
    "register(m, :mylog, 1, mylog; autodiff = true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "cbb79288-f5cc-40dc-8d30-045530f2c9f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "@NLobjective(m, Max, -sum(p[i] * mylog(p[i]) for i in 1:6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "8a0277a7-dd89-4e25-b585-7bc5615b8919",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$ p_{1} + p_{2} + p_{3} + p_{4} + p_{5} + p_{6} = 1.0 $$"
      ],
      "text/plain": [
       "p[1] + p[2] + p[3] + p[4] + p[5] + p[6] = 1.0"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@constraint(m, sum(p[i] for i in 1:6) == 1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "3b0eab02-1ec2-4592-9918-f119fed75a95",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in 1:6\n",
    "    @NLconstraint(m, p[i] >= 0.0)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "4436c864-d3e9-4269-989c-59844d6e02d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max -((p[1] * mylog(p[1]) + p[2] * mylog(p[2]) + p[3] * mylog(p[3]) + p[4] * mylog(p[4]) + p[5] * mylog(p[5]) + p[6] * mylog(p[6])))\n",
      "Subject to\n",
      " p[1] + p[2] + p[3] + p[4] + p[5] + p[6] = 1.0\n",
      " p[1] - 0.0 ≥ 0\n",
      " p[2] - 0.0 ≥ 0\n",
      " p[3] - 0.0 ≥ 0\n",
      " p[4] - 0.0 ≥ 0\n",
      " p[5] - 0.0 ≥ 0\n",
      " p[6] - 0.0 ≥ 0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "println(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "9e8e4aaf-9178-4a34-bb23-2fdff81b7d15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is Ipopt version 3.14.4, running with linear solver MUMPS 5.4.1.\n",
      "\n",
      "Number of nonzeros in equality constraint Jacobian...:        6\n",
      "Number of nonzeros in inequality constraint Jacobian.:        6\n",
      "Number of nonzeros in Lagrangian Hessian.............:        6\n",
      "\n",
      "Total number of variables............................:        6\n",
      "                     variables with only lower bounds:        0\n",
      "                variables with lower and upper bounds:        0\n",
      "                     variables with only upper bounds:        0\n",
      "Total number of equality constraints.................:        1\n",
      "Total number of inequality constraints...............:        6\n",
      "        inequality constraints with only lower bounds:        6\n",
      "   inequality constraints with lower and upper bounds:        0\n",
      "        inequality constraints with only upper bounds:        0\n",
      "\n",
      "iter    objective    inf_pr   inf_du lg(mu)  ||d||  lg(rg) alpha_du alpha_pr  ls\n",
      "   0 -0.0000000e+00 1.00e+00 0.00e+00  -1.0 0.00e+00    -  0.00e+00 0.00e+00   0\n",
      "   1  1.7917595e+00 1.11e-16 1.37e+01  -1.7 1.67e-01    -  6.75e-02 1.00e+00f  1\n",
      "   2  1.7917595e+00 0.00e+00 2.00e-07  -1.7 7.06e-17    -  1.00e+00 1.00e+00   0\n",
      "   3  1.7917595e+00 2.22e-16 1.50e-09  -3.8 7.20e-17    -  1.00e+00 1.00e+00   0\n",
      "   4  1.7917595e+00 1.11e-16 1.84e-11  -5.7 5.93e-17    -  1.00e+00 1.00e+00   0\n",
      "   5  1.7917595e+00 2.22e-16 2.51e-14  -8.6 1.85e-17    -  1.00e+00 1.00e+00T  0\n",
      "\n",
      "Number of Iterations....: 5\n",
      "\n",
      "                                   (scaled)                 (unscaled)\n",
      "Objective...............:  -1.7917594692280550e+00    1.7917594692280550e+00\n",
      "Dual infeasibility......:   2.5059036063649481e-14    2.5059036063649481e-14\n",
      "Constraint violation....:   2.2204460492503131e-16    2.2204460492503131e-16\n",
      "Variable bound violation:   0.0000000000000000e+00    0.0000000000000000e+00\n",
      "Complementarity.........:   2.5059035596800899e-09    2.5059035596800899e-09\n",
      "Overall NLP error.......:   2.5059035596800899e-09    2.5059035596800899e-09\n",
      "\n",
      "\n",
      "Number of objective function evaluations             = 6\n",
      "Number of objective gradient evaluations             = 6\n",
      "Number of equality constraint evaluations            = 6\n",
      "Number of inequality constraint evaluations          = 6\n",
      "Number of equality constraint Jacobian evaluations   = 6\n",
      "Number of inequality constraint Jacobian evaluations = 6\n",
      "Number of Lagrangian Hessian evaluations             = 5\n",
      "Total seconds in IPOPT                               = 0.053\n",
      "\n",
      "EXIT: Optimal Solution Found.\n"
     ]
    }
   ],
   "source": [
    "optimize!(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "de4c7066-f42b-4a08-b418-122b68542f62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6-element Vector{Float64}:\n",
       " 0.16666666666666669\n",
       " 0.16666666666666669\n",
       " 0.16666666666666669\n",
       " 0.16666666666666669\n",
       " 0.16666666666666669\n",
       " 0.16666666666666669"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "value.(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1f67070-0020-4111-a192-a81d97c01bbf",
   "metadata": {},
   "source": [
    "### Maximum Entropy For a Single Dice"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b974fd9c-f06b-4bf4-bf71-aec34303158b",
   "metadata": {},
   "source": [
    "$$\n",
    "\\begin{aligned}\n",
    "\\max z & = - \\sum_{i=1}^{n} p_i \\times \\log p_i \\\\\n",
    "\\text{Subject to:} \\\\\n",
    "& \\sum_{i=1}^{n} p_i = 1 \\\\\n",
    "& 0 \\le p_i \\le 1 \\\\\n",
    "& i = 1, 2, \\ldots, 6 \\\\\n",
    "\\end{aligned}\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1544429e-8a06-4ce4-a8a5-9be55537744a",
   "metadata": {},
   "source": [
    "### Solution"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b310fef-e46a-4597-bf6c-969fde1ed1d0",
   "metadata": {},
   "source": [
    "Entropy is maximum when $p_i = \\frac{1}{6}$ for $i = 1,2, \\ldots, 6$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3919e1c7-349a-4794-882a-65be27086310",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.7.2",
   "language": "julia",
   "name": "julia-1.7"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}