{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# A simple example demonstrating Multinomial HMM\n\nThe Multinomial HMM is a generalization of the Categorical HMM, with some key\ndifferences:\n\n- a Categorical__ (or generalized Bernoulli/multinoulli) distribution models an\n  outcome of a die with `n_features` possible values, i.e. it is a\n  generalization of the Bernoulli distribution where there are ``n_features``\n  categories instead of the binary success/failure outcome; a Categorical HMM\n  has the emission probabilities for each component parametrized by Categorical\n  distributions.\n\n- a Multinomial__ distribution models the outcome of ``n_trials`` independent\n  rolls of die, each with ``n_features`` possible values; i.e.\n\n  - when ``n_trials = 1`` and ``n_features = 1``, it is a Bernoulli\n    distribution,\n  - when ``n_trials > 1`` and ``n_features = 2``, it is a Binomial\n    distribution,\n  - when ``n_trials = 1`` and ``n_features > 2``, it is a Categorical\n    distribution.\n\nThe emission probabilities for each component of a Multinomial HMM are\nparameterized by Multinomial distributions.\n\n__ https://en.wikipedia.org/wiki/Categorical_distribution\n__ https://en.wikipedia.org/wiki/Multinomial_distribution\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "import numpy as np\nfrom hmmlearn import hmm\n\n# For this example, we will model the stages of a conversation,\n# where each sentence is \"generated\" with an underlying topic, \"cat\" or \"dog\"\nstates = [\"cat\", \"dog\"]\nid2topic = dict(zip(range(len(states)), states))\n# we are more likely to talk about cats first\nstart_probs = np.array([0.6, 0.4])\n\n# For each topic, the probability of saying certain words can be modeled by\n# a distribution over vocabulary associated with the categories\n\nvocabulary = [\"tail\", \"fetch\", \"mouse\", \"food\"]\n# if the topic is \"cat\", we are more likely to talk about \"mouse\"\n# if the topic is \"dog\", we are more likely to talk about \"fetch\"\nemission_probs = np.array([[0.25, 0.1, 0.4, 0.25],\n                           [0.2, 0.5, 0.1, 0.2]])\n\n# Also assume it's more likely to stay in a state than transition to the other\ntrans_mat = np.array([[0.8, 0.2], [0.2, 0.8]])\n\n\n# Pretend that every sentence we speak only has a total of 5 words,\n# i.e. we independently utter a word from the vocabulary 5 times per sentence\n# we observe the following bag of words (BoW) for 8 sentences:\nobservations = [[\"tail\", \"mouse\", \"mouse\", \"food\", \"mouse\"],\n        [\"food\", \"mouse\", \"mouse\", \"food\", \"mouse\"],\n        [\"tail\", \"mouse\", \"mouse\", \"tail\", \"mouse\"],\n        [\"food\", \"mouse\", \"food\", \"food\", \"tail\"],\n        [\"tail\", \"fetch\", \"mouse\", \"food\", \"tail\"],\n        [\"tail\", \"fetch\", \"fetch\", \"food\", \"fetch\"],\n        [\"fetch\", \"fetch\", \"fetch\", \"food\", \"tail\"],\n        [\"food\", \"mouse\", \"food\", \"food\", \"tail\"],\n        [\"tail\", \"mouse\", \"mouse\", \"tail\", \"mouse\"],\n        [\"fetch\", \"fetch\", \"fetch\", \"fetch\", \"fetch\"]]\n\n# Convert \"sentences\" to numbers:\nvocab2id = dict(zip(vocabulary, range(len(vocabulary))))\ndef sentence2counts(sentence):\n    ans = []\n    for word, idx in vocab2id.items():\n        count = sentence.count(word)\n        ans.append(count)\n    return ans\n\nX = []\nfor sentence in observations:\n    row = sentence2counts(sentence)\n    X.append(row)\n\ndata = np.array(X, dtype=int)\n\n# pretend this is repeated, so we have more data to learn from:\nlengths = [len(X)]*5\nsequences = np.tile(data, (5,1))\n\n\n# Set up model:\nmodel = hmm.MultinomialHMM(n_components=len(states),\n        n_trials=len(observations[0]),\n        n_iter=50,\n        init_params='')\n\nmodel.n_features = len(vocabulary)\nmodel.startprob_ = start_probs\nmodel.transmat_ = trans_mat\nmodel.emissionprob_ = emission_probs\nmodel.fit(sequences, lengths)\nlogprob, received = model.decode(sequences)\n\nprint(\"Topics discussed:\")\nprint([id2topic[x] for x in received])\n\nprint(\"Learned emission probs:\")\nprint(model.emissionprob_)\n\nprint(\"Learned transition matrix:\")\nprint(model.transmat_)\n\n# Try to reset and refit:\nnew_model = hmm.MultinomialHMM(n_components=len(states),\n        n_trials=len(observations[0]),\n        n_iter=50, init_params='ste')\n\nnew_model.fit(sequences, lengths)\nlogprob, received = new_model.decode(sequences)\n\nprint(\"\\nNew Model\")\nprint(\"Topics discussed:\")\nprint([id2topic[x] for x in received])\n\nprint(\"Learned emission probs:\")\nprint(new_model.emissionprob_)\n\nprint(\"Learned transition matrix:\")\nprint(new_model.transmat_)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}