%+
% Name:
%    memo_acis_detect_hotpix_1.0.tex
%
% Usage:
%    latex memo_acis_detect_hotpix_1.0.tex
%
% Description:
%    This document describes the specification of a tool designed to detect 
%    hot or flaring pixels.
%
% Input:
%
% Output:
%
% Notes:
%
% History:
%    27 Apr 02, created (v1.0), Glenn E. Allen
%-

\documentclass{article}

\usepackage{cxo-memo-logo}
\usepackage{draftcopy}
\usepackage[dvips]{graphics}
\usepackage{gea}

\newcommand{\adh}{acis\_detect\_hotpix}
\newcommand{\arraycol}{A_{\rm col}^{i}}
\newcommand{\arraycolnode}{A_{\rm col}^{ij}}
\newcommand{\arraypix}{A_{\rm pix}^{i}}
\newcommand{\arraypixnode}{A_{\rm pix}^{ij}}
\newcommand{\bitcol}{14}
\newcommand{\bitpix}{15}
\newcommand{\ignoreval}{-4096}
\newcommand{\ignorevalcol}{-4094}
\newcommand{\ignorevalpix}{-4095}
\newcommand{\meancolnode}{\mu_{\rm col}^{ij}}
\newcommand{\meanpixnode}{\mu_{\rm pix}^{ij}}
\newcommand{\ncolnode}{N_{\rm col}^{ij}}
\newcommand{\ncoltot}{N_{\rm col}}
\newcommand{\npixnode}{N_{\rm pix}^{ij}}
\newcommand{\npixtot}{N_{\rm pix}}
\newcommand{\probthreshcol}{P_{\rm col}}
\newcommand{\probthreshmax}{1.0e$-$1}
\newcommand{\probthreshmin}{1.0e$-$10}
\newcommand{\probthreshpix}{P_{\rm pix}}
\newcommand{\regwidthmax}{256}
\newcommand{\regwidthmin}{3}
\newcommand{\sigmathreshcol}{\sigma_{\rm col}}
\newcommand{\sigmathreshpix}{\sigma_{\rm pix}}

\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 1. Header

\memobasic{
   Martin Elvis, SDS Group Leader }{
   Glenn E.\ Allen, SDS }{
   \adh }{
   1.0 }{
   http://space.mit.edu/CXC/docs/docs.html\#hotpix }{
   /nfs/cxc/h2/gea/sds/docs/memos/memo\_\adh\_1.0.tex }

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 2. Tool name

\section{\adh}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 3. Description

\subsection{ Description }

This tool uses the event data or bias data for a given observation to search
for pixels and columns that may be temporarily or permanently ``bad.'' The
tool is designed to find pixels that are not included in the bad pixel file
for the observation.  Known bad pixels and columns (and masked regions) can
be excluded from the search.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 4. Input

\subsection{ Input }

\begin{enumerate}

\item
A Level 1 event data file (acis*evt1.fits) or a Level 0 bias map
(acis*bias0.fits)

\item
A Level 1 bad pixel file (acis*bpix1.fits)

\item
A Level 1 window mask file (acis*msk1.fits)

\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 5. Output

\subsection{ Output }

The output file has the same format as a Level 1 bad pixel file.  If an
input bad pixel file is specified, the output is the union of this file and
any newly identified bad pixels and columns.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 6. Parameters

\subsection{ Parameters }

\begin{enumerate}

\item
infile,s,a,``'',,,``Name of input event file or bias files''

\item
outfile,s,a,``'',,,``Name of output bad pixel file''

\item
badpixfile,s,a,``CALDB'',,,``Name of input bad pixel file (CALDB |
$\langle$filename$\rangle$ | none | NONE)''

\item
maskfile,s,a,``default'',,,``Name of input mask file (default |
$\langle$filename$\rangle$ | none | NONE)''

\item
logfile,s,h,``stdout'',,,``Destination of output messages (stdout | STDOUT
| $\langle$filename$\rangle$)''

\item
probthresh,r,h,1.0e-3,\probthreshmin,\probthreshmax,``Minimum significance of 
potentially bad pixels in sigma''

\item
regwidth,i,h,5,\regwidthmin,\regwidthmax,``Size of reference region for 
comparison (\eg\ 5x5)''

\item
clobber,b,h,no,,,``Overwrite output file if it already exists?''

\item
verbose,i,h,0,0,5,``Amount of messages produced (0=none, 5=most)''

\item
mode,s,h,``ql'',,,

\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 7. Processing

\subsection{ Processing }

\begin{enumerate}

\item
Verify that the specified input files exist.  If the parameter ${\rm
clobber} = {\rm ``no}$,'' verify that the output file does not exist. Verify
that the values specified for the parameters probthresh and regwidth are in
the allowed ranges from \probthreshmin\ to \probthreshmax\ and from
\regwidthmin\ to \regwidthmax, respectively.

\item
Create a $1024 \times 1024$ element array $\arraypix$ and a $1 \times 1024$
element array $\arraycol$ for each active CCD $i$.

\item
Read the input bad pixel file (if one is specified).  If the parameter
badpixfile $=$ ``CALDB,'' then use the appropriate bad pixel ARD file.  If a
pixel or column is designated as ``bad,'' then set the corresponding
elements of $\arraypix$ to be $\ignoreval$.  If a column is designated as
``bad,'' then set the corresponding element of $\arraycol$ to be
$\ignoreval$.  Do not set the elements of $\arraypix$ and $\arraycol$ $=$
$\ignoreval$ if the only STATUS bit(s) set equal to one in the bad pixel
file are 5, 6, or 8--12.  Since pixels and columns that have only these bits
set are not truly bad, they should be included in the search.

\item
Read the input mask file (if one is specified).  If the parameter maskfile
$=$ ``default,'' then use the appropriate mask specified in the
acis\_build\_mask document at http://space.mit.edu/CXC/docs/docs.html\#msk.
If a portion of a CCD is ``masked out'' (\ie\ no events can be reported for
the region), then set the values of the appropriate elements of $\arraypix$
and $\arraycol$ to be $\ignoreval$.

\item
If the input file specified by the parameter infile is an event data file,
then read the values of CCD\_ID, CHIPX, and CHIPY for the first event and
add one to the corresponding elements of $\arraypix$ and $\arraycol$ (unless
the value of $\arraypix$ or $\arraycol = \ignoreval$).  Repeat this process
for the rest of the events in the file.  If the file(s) specified by infile
is a bias map(s), then read the value of the first element of the first bias
map, set the corresponding value of $\arraypix$ equal to the value in the
bias map (unless the value of $\arraypix = \ignoreval$), and add the value
in the bias map to the appropriate element of $\arraycol$ (unless the value
of $\arraycol = \ignoreval$).  If the value in the bias map is greater than
or equal to 4094, then set the corresponding element of $\arraypix =
\ignoreval$.  The values in the elements of the array $\arraycol$ are sums
over all of the pixels in the corresponding CCD column.

\item
Compute the total number of pixels and columns to be searched.  Set
$\npixtot$ equal to the total number of pixels for which $\arraypix \ne
\ignoreval$.  If six CCDs are active and the default window masks are used
for the observation, $\npixtot \approx 6 \times 10^{6}$.  Set $\ncoltot$
equal to the total number of columns for which $\arraycol \ne \ignoreval$.
If six CCDs are active, $\ncoltot \approx 6 \times 10^{3}$.  Write the
values of $\npixtot$ and $\ncoltot$ to the logfile (for some value of
verbose).

\item
Compute the minimum significance of the pixels and columns identified as
bad.  This significance depends on the total number of pixels or columns
searched (\ie\ the number of trials) and the specified value of the
parameter probthresh.  The probability threshold for the search for bad
pixels
%
\begin{equation}
\probthreshpix
   \approx
   \frac{ {\rm probthresh} }{ \npixtot }
\end{equation}
%
and the probability threshold for the search for bad columns
%
\begin{equation}
\probthreshcol
   \approx
   \frac{ {\rm probthresh} }{ \ncoltot }.
\end{equation}
%
If the input file specified by the parameter infile is an event data file,
then the corresponding significance thresholds in terms of the parameter
$\sigma$ of the Gaussian function $f ( x ) = \frac{ 1 }{ \sqrt{ 2 \pi }
\sigma } e^{ - x^{2} / 2 \sigma^{2} }$ are obtained by solving the equations
%
\begin{equation}
\probthreshpix 
   =
   {\rm erfc} \left( \sigmathreshpix \right)
   \label{eqn3}
\end{equation}
%
and
%
\begin{equation}
\probthreshcol
   =
   {\rm erfc} \left( \sigmathreshcol \right)
   \label{eqn4}
\end{equation}
%
for $\sigmathreshpix$ and $\sigmathreshcol$.  For example, if $\npixtot = 6
\times 10^{6}$ and ${\rm probthresh} = 10^{-3}$, then $\probthreshpix = 1.67
\times 10^{-10}$ and $\sigmathreshpix = 6.39$.  If $\ncoltot = 6 \times
10^{3}$ and ${\rm probthresh} = 10^{-3}$, then $\probthreshcol = 1.67 \times
10^{-7}$ and $\sigmathreshcol = 5.23$.  Write the values of
$\probthreshpix$, $\probthreshcol$, $\sigmathreshpix$, and $\sigmathreshcol$
to the log file (for some value of verbose).

\item
Begin the search for bad pixels.  Perform the search separately for each
node of each active CCD because the event rate and bias can vary from node
to node on the same CCD.

\item
\label{step9}
If the file specified by the input parameter infile is an event data file,
then compute the {\it mean} value of the number of events per pixel
%
\begin{equation}
\meanpixnode
   = \frac{ \Sigma \arraypixnode }{ \npixnode }
\end{equation}
%
The sum is performed over all valid pixels of node $j$ on CCD $i$.
$\npixnode$ is the total number of valid pixels on the node.  Here, invalid
pixels are those for which the values of $\arraypix = \ignoreval$. Note that
the mean is a real number, not an integer.

If the file(s) specified by the input parameter infile is a bias map(s),
then compute the {\it median} value of the bias:
%
\begin{equation}
\meanpixnode
   = {\rm median} \left( \arraypixnode \right).
\end{equation}
%
Only include the valid pixels of node $j$ on CCD $i$.  Here, invalid pixels
are those for which the values of $\arraypix = \ignoreval$.

Write the value of $\meanpixnode$ to the logfile (for some value of
verbose).

\item
\label{step10}
If the file(s) specified by the input parameter infile is a bias map(s),
then subtract the value of $\meanpixnode$ from all of the valid elements of
$\arraypix$ (on the same node).  Do not perform the subtraction if the value
of an element $\arraypixnode \le \ignorevalpix$.

\item
\label{step11}
If the file(s) specified by the parameter infile is a bias map(s), then 
compute the minimum significance of the pixels identified as bad:
%
\begin{equation}
\sigmathreshpix
   \approx
   \left( 
      \frac{ 
         \Sigma 
         \left( \arraypixnode - \left< \arraypixnode \right> \right)^{2}
      }{ 
         \npixnode } 
         \right)^{ \frac{1}{2} }
   x_{\rm pix}
\end{equation}
%
where $\left< \arraypixnode \right>$ is the mean (real-numbered) value of
the valid elements of the pixel array and $x_{\rm pix}$ is the solution of
$\probthreshpix = {\rm erfc}(x_{\rm pix})$. The sum is performed only over
the valid elements of the array. $\npixnode$ is the total number of valid
pixels on the node.  Here, the invalid pixels are those for which the values
of $\arraypix \le \ignorevalpix$ and those for which $\left| \arraypixnode -
\left< \arraypixnode \right> \right| > 3$.

\item
\label{step12}
For each valid pixel of a node, determine the value of the corresponding
element of the pixel array ($S$, \ie\ the ``source'' value), the value to
use as a reference ($B$, \ie\ the ``background'' value), and the number of
background pixels used to compute the reference value ($n$).
%
\begin{equation}
S
   =
   {\rm the\ value\ of\ the\ appropriate\ element\ of}\ 
   \arraypixnode.
\end{equation}
%
\begin{equation}
n
   =
   {\rm the\ number\ of\ valid\ pixels\ to\ use\ to\ compute}\ B.
\end{equation}
%
Typically, $n = {\rm regwidth} \times {\rm regwidth} - 1$, but the value of
$n$ is smaller if one or more of the pixels in the reference region are
invalid.  The reference region is a square ``centered'' on the source.  For
example, if ${\rm regwidth} = 5$ and the pixel being examined is $\chipx =
372$ and $\chipy = 948$, then the reference region is the region defined by
$370 \le \chipx \le 374$ and $946 \le \chipy \le 950$.  If ${\rm regwidth} =
6$, then the reference region for the same pixel is $370 \le \chipx \le 375$
and $946 \le \chipy \le 951$.  The pixel being examined is excluded from the
computation of $n$ and $B$.  Any other pixels in the reference region for
which $\arraypixnode \le \ignorevalpix$ are also excluded from the
computation of $n$ and $B$.  The reference region may not extend off of the
node. For example, if the pixel being considered is $\chipx = 512$ and
$\chipy = 2$, then the reference region is $510 \le \chipx \le 512$ and $1
\le \chipy \le 4$ (if ${\rm regwidth} = 5$).  If there are no valid pixels
in the reference region or if ${\rm regwidth} = 256$, then $n$ is equal to
the total number of valid pixels on the entire node.

%
\begin{equation}
B
   =
   \Sigma \arraypixnode,
\end{equation}
%
where the sum is performed over the $n$ valid elements of the reference
region.

\item
If the file specified by the parameter infile is an event data file, then
the statistical significance of $S$ for a pixel given the values of $B$ and
$n$ for the pixel (\ie\ the likelihood that the value of $\arraypixnode$ for
the pixel is significantly larger or smaller than the values of
$\arraypixnode$ for the reference pixels) is
%
\begin{equation}
\sigma
   =
   2^{ \frac{1}{2} }
   \left(
      S
      \ln
      \left(
         \frac{ \left( n+1 \right) S }{ S+B }
      \right)
      +
      B
      \ln
      \left(
         \frac{ n+1 }{ n }
	 \frac{ B }{ S+B }
      \right)
   \right)^{ \frac{1}{2} }
\end{equation}
%
(Li and Ma (1983)).

If the file specified by the parameter infile is a bias map, then the
statistical significance of $S$ for a pixel given the values of $B$ and $n$
for the pixel is
%
\begin{equation}
\sigma
   =
   S - \frac{ B }{ n }
\end{equation}
%

\item
\label{step14} 
If the absolute value of the significance $\left| \sigma \right| \ge
\sigmathreshpix$, then flag the pixel as potentially bad.  Set the value of
$\arraypixnode = \ignorevalpix$.

\item
\label{step15}
Repeat steps~\ref{step9}--\ref{step14} for each valid pixel of each node of
each active CCD.

\item
Begin the search for bad columns.  The procedure used to search for bad
columns is very similar to the procedure used to search for bad pixels
(steps~\ref{step9}--\ref{step15}) with appropriate substitutions of
$\arraycol$, $\arraycolnode$, $\meancolnode$, $\ncoltot$, and $x_{\rm col}$,
for $\arraypix$, $\arraypixnode$, $\meanpixnode$, $\npixtot$, and $x_{\rm
pix}$, respectively.  In steps~\ref{step10}, \ref{step11}, \ref{step12}, and
\ref{step14}, substitute the value $\ignorevalcol$ for the value
$\ignorevalpix$.

\item
Write the output to the file specified by the parameter outfile unless a
file by that name already exists and the parameter clobber $=$ ``no,'' in
which case write an error message to the logfile and exit.  The format of
the output file is the same as the format of an ACIS Level 1 bad pixel file.
If an input bad pixel file is specified by the parameter badpixfile, then
copy the data in the input file to the output file.  Newly identified bad
pixels and columns are added to the output file.  Bits \bitpix\ and \bitcol\
(0f 0-31) are set to one for the newly identified bad pixel and columns,
respectively.  For each new bad pixel or column, write the values of \ccdid,
\chipx, \chipy, $S$, $B$, $n$, and $\sigma$ to the logfile.  If no new bad
pixels or columns are identified, do not create an output file and write an
appropriate message to the logfile.

\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 8. References

\subsection{ References }

Grant, C.\ 2002, private communication

\noindent
Isobe, T.\ 2002, private communication

\noindent
Li, T.-P., and Ma, Y.-Q.\ 1983, \apj, 272, 317

\noindent
Tibbetts, K.\ 2002, private communication

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 9. Finish

\end{document}

