%+
% Name:
%   deltoclk_spec_1.4.tex
%
% History:
%   2011 Jun 03, created (v1.0), Glenn E. Allen
%   2011 Jun 09, removed mention of acis_format_events; added the
%     exposure-statistics file to the list of input files; added the
%     parameter minnumframes; added the definition of a valid frame; added a
%     description of what to do if frames are missing; added a definition of
%     median for even and odd numbers of elements; added the section "TBD"
%     (v1.1), GEA
%   2011 Jun 12, added the package epsfig; added Figures 1, 2, and 3 (v1.2),
%     GEA
%   2011 Jun 17, tried to clarify the conditions in equations 2-5; added
%     a figure of examples of anomalous DELTOCLKs (v1.3), GEA
%   2011 Nov 15, changed frame k to frame k+1 following a discussion with
%     Peter; use the median instead of the actual value of OVRCLOCK to
%     compute PHAS for frames with anomalous values of OVRCLOCK (v1.4), GEA
%-

\documentclass{article}

\usepackage{cxo-memo-logo}
\usepackage{epsfig}
\usepackage{gea}
\usepackage{pstricks}

\newrgbcolor{blue}{0 0 1}
\newrgbcolor{red}{1 0 0}

\renewcommand{\thefootnote}{\fnsymbol{footnote}}

\begin{document}

% 1. Header

\memobasic{
   Jonathan McDowell, SDS Group Leader }{
   Glenn E.\ Allen, SDS }{
   \tt DELTOCLK spec }{
   1.4 }{
   http://space.mit.edu/CXC/docs/docs.html\#deltoclk }{
   /nfs/cxc/h2/gea/sds/docs/memos/deltoclk\_spec\_1.4.tex }

% 2. Description

\section{ Description }

As shown in Figure~\ref{fig01}, occasionally the values of {\tt DELTOCLK} in
exposure-records files are anomalously high or anomalously low. Inaccurate
values of {\tt DELTOCLK} lead to inaccurate pulse heights. Therefore events
that are processed using anomalous values have a {\tt STATUS} bit set to one
and are discarded from Level 2 event-data files. Note that the input data
are the values of {\tt OVRCLOCK} from exposure-statistics files instead of
the values of {\tt DELTOCLK} from exposure-records files because the
difference between the two sets of values for a node is a constant (the
initial overclock) and because the exposure-statistics files are available
to users who may want to reprocess data themselves.

% 3. Input

\section{ Input }

\begin{enumerate}

  \item
    A Level 1 event-data file (acis*evt1.fits)

  \item
    An exposure-statistics file (acis*stat1.fits)

\end{enumerate}

% 4. Output

\section{ Output }

\begin{enumerate}

  \item
    An updated event-data file

  \item
    An updated exposure-statistics file

\end{enumerate}

% 5. Parameters

\section{ Parameters }

\begin{enumerate}

  \item
    { \tt infile,f,a,"",,,"Name of input event-data file" }

  \item
    { \tt outfile,f,a"",,,"Name of output event-data file" }

  \item
    { \tt expstatsfile,f,a,"",,,"Name of exposure-statistics file to be
    updated" }

  \item
    { \tt numframes,i,h,5,3,1001,"Nominal number of frames in the sliding
    time window (must be \\ odd)" }

  \item
    { \tt maxframegap,i,h,11}%
    %
    \footnote{The largest {\tt DTYCYCLE} used to date is 10. Furthermore, as
    shown in Figure~\ref{fig02}, consecutive frame numbers
    typically differ by eleven or less.}%
    %
    {\tt,1,1001,"Maximum difference between consecutive frame numbers in \\
    window" }

  \item
    { \tt minnumframes,i,h,3,3,1001,"Minimum number of frames in window" }

  \item
    { \tt deltoclkthresh,i,h,3}%
    %
    \footnote{As shown in Figures~\ref{fig03}--\ref{fig05}, the algorithm is
    relatively insensitive to the value of {\tt deltoclkthresh} for values
    greater than or equal to three.}%
    %
    {\tt,1,4096,"Minimum offset in adu that is considered anomalous" }

\end{enumerate}

% 6. Processing

\section{ Processing }

\begin{enumerate}

  \item
    Verify that the {\tt infile} and {\tt expstatsfile} exist.
    %
    If {\tt clobber=no}, then verify that the {\tt outfile} does not
    exist.
    %
    Verify that the values of the parameters {\tt numframes}, {\tt
    maxframegap}, {\tt minnumframes} and {\tt deltoclkthresh} are in the
    valid ranges for these parameters.
    %
    Note that {\tt numframes} must be an odd number.

  \item
    If the data are being reprocessed by a user, then perform the following
    two steps.
    %
    \begin{enumerate}

      \item
        Set {\tt STATUS} bit 10 (of 0-31) to zero (i.e.\ unset it) for all
	of the events in the {\tt outfile}.  Subsequent processing sets this
	bit to one for the appropriate events.

      \item
        Remove the contents of the HDU ``{\tt BADOCLK}'' in the {\tt
	expstatsfile}. This HDU is updated appropriately in subsequent
	processing.

    \end{enumerate}

  \item
    Perform the following steps separately for each node of each CCD.
    %
    \begin{enumerate}

      \item
        For each valid%
	%
	\footnote{Here, a frame is valid if:
	%
	\begin{itemize}
	  \item[]
	    the value of {\tt EXPNO} for the frame is greater than the value
	    of {\tt EXPNO} for the preceding frame (if any),
	  \item[]
	    the value of {\tt EXPNO} for the frame is less than the value of
	    {\tt EXPNO} for the subsequent frame (if any), and
	  \item[]
	    there is {\tt OVRCLOCK} data available for the frame.
	\end{itemize}
	%
	If one or more of these conditions are not satisfied, then the frame
	is considered invalid.  Invalid frames are not included in the
	``{\tt BADOCLK}'' HDU of the {\tt expstatsfile}.  Any events that
	may occur during invalid frames do not have {\tt STATUS} bit 10 set
	to one.}
	%
	frame $k$, compute the expected value $M_{k}$ of {\tt
	OVRCLOCK}.  In most cases, $M_{k}$ is the median%
	%
	\footnote{If the number of elements in the set for which the median
	is being computed is odd, then the median is the middle value of the
	set after the set has been sorted (e.g.\ the median of
	[1,1,1,4,5,6,7] is 4).  If the number of elements is even, then the
	median is the mean of the middle two values of the sorted set.  If
	the mean is not an integer, then the mean is truncated to obtain an
	integer (e.g.\ the median of [1,1,1,4,5,6] is 2, not 2.5).}
	%
	of the values of {\tt OVRCLOCK} for the frames from $k-n$ to $k+n$,
	inclusive, where
	%
	\begin{equation}
	  n = \frac{{\tt numframes} - 1}{2}.
	\end{equation}
	%
	There are some cases that require special treatment.
	%
	\begin{enumerate}

          \item
	    If one or more of the frames immediately before and/or
	    immediately after frame $k$ are invalid, then use the nearest
	    valid frames, not the invalid frames to compute the median.  For
	    example, if ${\tt numframes} = 5$, then the {\tt OVRCLOCK} data
	    for the consecutive valid frames $i$, $j$, $k$, $l$, and $m$ are
	    used to compute the median, provided that ${\tt EXPNO}_{i} <
	    {\tt EXPNO}_{j} < {\tt EXPNO}_{k} < {\tt EXPNO}_{l} < {\tt
	    EXPNO}_{m}$ and that
	    %
	    \begin{eqnarray}
	      {\tt EXPNO}_{j} - {\tt EXPNO}_{i} & \le & {\tt maxframegap},
	        \label{eqn01} \\
	      {\tt EXPNO}_{k} - {\tt EXPNO}_{j} & \le & {\tt maxframegap},
	        \label{eqn02} \\
	      {\tt EXPNO}_{l} - {\tt EXPNO}_{k} & \le & {\tt maxframegap},\
	        {\rm and} \label{eqn03} \\
	      {\tt EXPNO}_{m} - {\tt EXPNO}_{l} & \le & {\tt maxframegap}.
	        \label{eqn04}
	    \end{eqnarray}
	    %
	    If equation~\ref{eqn01} is not satisfied, then exclude the data
	    for frame ${i}$ from the computation of the median.  If
	    equation~\ref{eqn02} is not satisfied, then exclude the data for
	    frames $i$ and $j$.  Similarly, if equation~\ref{eqn03} is not
	    satisfied, then exclude the data for frames $l$ and $m$ and if
	    equation~\ref{eqn04} is not satisfied, then exclude the data for
	    frame $m$.  That is, in no circumstance should the data set
	    used to compute the median extend across a gap in frame numbers
	    that is larger than {\tt maxframegap}.

	  \item
	    At or near the beginning of the data, there may not be $n$ valid
	    frames preceding frame $k$.  For example, for the first valid
	    frame, there are no valid frames that precede it. In this case,
	    the median is computed using the data for the first frame and
	    for the $n$ subsequent, valid frames (provided that the
	    subsequent frames do not violate the {\tt maxframegap}
	    constraint).  For the second valid frame, the median is computed
	    using the data for the first and second valid frames and for the
	    $n$ subsequent, valid frames.  (Again, the {\tt maxframegap}
	    constraint applies.)

	  \item
	    Likewise, at or near the end of the data, there may not be $n$
	    valid frames following frame $k$.

	  \item
	    There must be at least ${\tt minnumframes}$ valid frames in the
	    set of data used to compute $M_{k}$.  If there are fewer than
	    ${\tt minnumframes}$ valid frames, due to violations of the {\tt
	    maxframegap} constraint and/or due to the lack of data at the
	    beginning or the end of a data set, then set $M_{k}$ equal to
	    the value of {\tt OVRCLOCK} for frame $k$ (i.e.\ there is too
	    little data to determine whether the value
	    of {\tt OVRCLOCK} is anomalous).%
	    %
	    \footnote{In this case, frame $k$ is not included in the ``{\tt
	    BADOCLK}'' HDU of the {\tt expstatsfile} and any events that
	    occur during frame $k$ do not have {\tt STATUS} bit 10 set to
	    one.}
	    %
	    A consequence of this condition is that an {\tt infile} must
	    have at least {\tt minnumframes} valid frames of data for a node
	    in order to search for anomalous values of {\tt OVRCLOCK} for
	    the node.

	\end{enumerate}

      \item
        { \red When the values of {\tt PHAS} are computed from the values of
	{\tt RAW\_PHAS}, the value of $M_{k}$, not the value of {\tt
	OVRCLOCK}, is used for frames with anomalous values of {\tt
	OVRCLOCK}. }

      \item
        If the value of {\tt OVRCLOCK} for a valid frame is greater than or
	equal to $M_{k} + {\tt deltoclkthresh}$ or is less than or equal to
	$M_{k} - {\tt deltoclkthresh}$, then the value of {\tt OVRCLOCK} is
	anomalous.  In this case,
	%
	\begin{enumerate}
	  \item
	    modify the {\tt outfile} to set {\tt STATUS} bit 10 (of 0-31) to
	    one for all of the events that occur during frame { \red $k+1$
	    (not frame $k$)%
	    \footnote{\red Onboard, the value of {\tt DELTOCLK} is computed
	    at the end of a frame, not the beginning, and is used in the
	    following frame.}
	    %
	    } for the particular CCD and node in question
	    and
	  \item
	    update the ``{\tt BADOCLK}'' HDU of the {\tt expstatsfile} to
	    include the {\tt CCD\_ID}, {\tt NODE\_ID}, and {\tt EXPNO}
	    information for the CCD, node, and frame in question { \red
	    (frame $k+1$, not frame $k$). }
	\end{enumerate}

    \end{enumerate}

\end{enumerate}

% 7. TBD

\section{ TBD }

\begin{enumerate}

  \item
    Recreate Figures \ref{fig03}--\ref{fig05} (especially \ref{fig05}) to
    determine what the default value of {\tt maxframegap} should be.

  \item
    Plot the number of anomalies as a function of {\tt TIME} and {\tt CCD\_ID}
    and {\tt NODE\_ID}.

\end{enumerate}

% 8. Figures

\newpage

% 8.1. Fig. 1

\begin{figure}
  \begin{center}
    \epsfig{file=plot_deltoclk_examples,angle=-90,width=6.5in}
  \end{center}
  \caption{%
    Three examples of the relative values of {\tt DELTOCLK} as a function of
    frame number.  The data are from exposure-statistics files and an
    arbitrary constant has been added to the values of {\tt DELTOCLK} to
    improve the visual clarity.  The black, red, and green curves are the
    data for $({\tt OBS\_ID}, {\tt CCD\_ID}, {\tt NODE\_ID}) = (1886,7,0)$,
    (11793,9,1), and (13019,9,2), respectively.  Anomalous ``spikes'' are
    evident in the red and green curves.  The black curve has no such
    anomalies.
    %
    \label{fig01}
    }
\end{figure}

% 8.2. Fig. 2

\begin{figure}
  \begin{center}
    \epsfig{file=plot_expno_diffs,angle=0,width=6.5in}
  \end{center}
  \caption{%
    A histogram of the differences between consecutive frame numbers for
    63,778 exposure-records files.  A separate histogram was computed for
    each node of each file.  The histogram in this figure is a sum of each
    of these histograms.  If a difference is greater than 1000, then the
    difference is set to 1000.  If a difference is less than 0, then the
    difference is set to 0.  The inset histogram is a plot of the same data,
    except that it includes only the data in the range from 0 to 100.  The
    histogram includes all frames, whether they are valid or not.
    %
    \label{fig02}
    }
\end{figure}

% 8.3. Fig. 3

\begin{figure}
  \begin{center}
    \epsfig{file=plot_deltoclk_diffsA,angle=-90,width=6.5in}
  \end{center}
  \caption{%
    A histogram of the differences between the values of {\tt DELTOCLK} and
    the values of the median {\tt DELTOCLK} for 63,778 exposure-records
    files.  A separate histogram was computed for each node of each file.
    The histogram in this figure is a sum of each of these histograms.  The
    values of the median were computed using a ${\tt numframes} = 5$, ${\tt
    maxframegap} = \infty$, and ${\tt minnumframes} = 3$.  If a value of
    {\tt DELTOCLK} differs by more than 4096 adu from the corresponding
    median, then the difference is set to 4096 or $-4096$ as appropriate.
    %
    \label{fig03}
    }
\end{figure}

% 8.4. Fig. 4

\begin{figure}
  \begin{center}
    \epsfig{file=plot_deltoclk_diffsB,angle=-90,width=6.5in}
  \end{center}
  \caption{%
    This figure is identical to Figure~\ref{fig03}, except that it includes
    only the data in the range from -100 to 100 adu.
    %
    \label{fig04}
    }
\end{figure}

% 8.5. Fig. 5

\begin{figure}
  \begin{center}
    \epsfig{file=plot_anom_frac,angle=0,width=6.5in}
  \end{center}
  \caption{%
    A plot of the fraction of the values of {\tt DELTOCLK} that are
    identified as anomalous as a function of the parameter {\tt
    deltoclkthresh}.  This plot was produced using the data that is shown in
    Figure~\ref{fig02}.  The inset histogram is a plot of the same data,
    except that it includes only the data in the range from 0 to 20.
    %
    \label{fig05}
    }
\end{figure}

% 9. Finish

\end{document}

