Q1
← Back
Basic Info
Probability
└── Homework
└── W10
└── Q1.tex
Preview
\documentclass[12pt]{article}
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage[margin=1in]{geometry}
\usepackage{fancyhdr}
\usepackage{enumerate}
\usepackage[shortlabels]{enumitem}
\pagestyle{fancy}
\fancyhead[l]{Li Yifeng}
\fancyhead[c]{Homework \#10}
\fancyhead[r]{\today}
\fancyfoot[c]{\thepage}
\renewcommand{\headrulewidth}{0.2pt}
\setlength{\headheight}{15pt}
\newcommand{\bE}{\mathbb{E}}
\newcommand{\bP}{\mathbb{P}}
\begin{document}
\section*{Question 1}
\noindent
\bigskip
\begin{enumerate}[start=1,label={\bfseries Part \arabic*:},leftmargin=0in]
\bigskip\item
\subsection*{Solution}
First compute the marginal distribution of $X$:
\[
P(X=0)=\frac18,\quad P(X=1)=\frac78.
\]
Then
\[
H(X)=-\sum_{x=0}^1 P(X=x)\log_2P(X=x)
=-\Bigl(\tfrac18\log_2\tfrac18+\tfrac78\log_2\tfrac78\Bigr)
\approx0.544\text{ bits}.
\]
Next compute the conditional entropy:
\[
H(X\mid Y)
=\sum_{y=0}^1P(Y=y)\,H(X\mid Y=y).
\]
For $Y=0$, $P(Y=0)=\tfrac18$ and $X$ is degenerate ($H=0$).\\
For $Y=1$, $P(Y=1)=\tfrac78$ and
\[
P(X=0\mid1)=\frac{1/8}{7/8}=\frac17,\quad
P(X=1\mid1)=\frac67,
\]
so
\[
H(X\mid Y=1)
=-\Bigl(\tfrac17\log_2\tfrac17+\tfrac67\log_2\tfrac67\Bigr)
\approx0.592\text{ bits}.
\]
Hence
\[
H(X\mid Y)
=\frac18\cdot0+\frac78\cdot0.592
\approx0.519\text{ bits}.
\]
Finally, compare:
\[
H(X)\approx0.544\;\ge\;0.519\approx H(X\mid Y).
\]
\subsection*{Answer}
\[
\boxed{
H(X)\approx0.544\text{ bits},\quad
H(X\mid Y)\approx0.519\text{ bits},\quad
H(X)\ge H(X\mid Y).}
\]
\bigskip\item
\subsection*{Solution}
We compare $H(X\mid Y=1)\approx0.592$ bits (from above) with the a‐priori $H(X)\approx0.544$ bits.
\subsection*{Answer}
\[
\boxed{
H(X\mid Y=1)\approx0.592\text{ bits}
\;>\;
H(X)\approx0.544\text{ bits}.
}
\]
\bigskip\item
\subsection*{Solution}
The mutual information is
\[
I(X;Y)=H(X)-H(X\mid Y)
\approx0.544-0.519=0.025\text{ bits}.
\]
\subsection*{Answer}
\[
\boxed{I(X;Y)\approx0.025\text{ bits}.}
\]
\end{enumerate}
\end{document}