%+
% Name:
% deltoclk_spec_1.3.tex
%
% History:
% 2011 Jun 03, created (v1.0), Glenn E. Allen
% 2011 Jun 09, removed mention of acis_format_events; added the
% exposure-statistics file to the list of input files; added the
% parameter minnumframes; added the definition of a valid frame; added a
% description of what to do if frames are missing; added a definition of
% median for even and odd numbers of elements; added the section "TBD"
% (v1.1), GEA
% 2011 Jun 12, added the package epsfig; added Figures 1, 2, and 3 (v1.2),
% GEA
% 2011 Jun 17, tried to clarify the conditions in equations 2-5; added
% a figure of examples of anomalous DELTOCLKs (v1.3), GEA
%-
\documentclass{article}
\usepackage{cxo-memo-logo}
\usepackage{epsfig}
\usepackage{gea}
\usepackage{pstricks}
\newrgbcolor{blue}{0 0 1}
\newrgbcolor{red}{1 0 0}
\renewcommand{\thefootnote}{\fnsymbol{footnote}}
\begin{document}
% 1. Header
\memobasic{
Jonathan McDowell, SDS Group Leader }{
Glenn E.\ Allen, SDS }{
\tt DELTOCLK spec }{
1.3 }{
http://space.mit.edu/CXC/docs/docs.html\#deltoclk }{
/nfs/cxc/h2/gea/sds/docs/memos/deltoclk\_spec\_1.3.tex }
% 2. Description
\section{ Description }
{\red As shown in Figure~\ref{fig01},} occasionally the values of {\tt
DELTOCLK} in exposure-records files are anomalously high or anomalously low.
Inaccurate values of {\tt DELTOCLK} lead to inaccurate pulse heights.
Therefore events that are processed using anomalous values have a {\tt
STATUS} bit set to one and are discarded from Level 2 event-data files.
Note that the input data are the values of {\tt OVRCLOCK} from
exposure-statistics files instead of the values of {\tt DELTOCLK} from
exposure-records files because the difference between the two sets of values
for a node is a constant (the initial overclock) and because the
exposure-statistics files are available to users who may want to reprocess
data themselves.
% 3. Input
\section{ Input }
\begin{enumerate}
\item
A Level 1 event-data file (acis*evt1.fits)
\item
An exposure-statistics file (acis*stat1.fits)
\end{enumerate}
% 4. Output
\section{ Output }
\begin{enumerate}
\item
An updated event-data file
\item
An updated exposure-statistics file
\end{enumerate}
% 5. Parameters
\section{ Parameters }
\begin{enumerate}
\item
{ \tt infile,f,a,"",,,"Name of input event-data file" }
\item
{ \tt outfile,f,a"",,,"Name of output event-data file" }
\item
{ \tt expstatsfile,f,a,"",,,"Name of exposure-statistics file to be
updated" }
\item
{ \tt numframes,i,h,5,3,1001,"Nominal number of frames in the sliding
time window (must be \\ odd)" }
\item
{ \tt maxframegap,i,h,11}%
%
\footnote{The largest {\tt DTYCYCLE} used to date is 10. Furthermore, as
shown in Figure~\ref{fig03}, consecutive frame numbers
typically differ by eleven or less.}%
%
{\tt,1,1001,"Maximum difference between consecutive frame numbers in \\
window" }
\item
{ \tt minnumframes,i,h,3,3,1001,"Minimum number of frames in window" }
\item
{ \tt deltoclkthresh,i,h,3}%
%
\footnote{As shown in Figure~\ref{fig05}, the algorithm is relatively
insensitive to the value of {\tt deltoclkthresh} for values
greater than or equal to three.}%
%
{\tt,1,4096,"Minimum offset in adu that is considered anomalous" }
\end{enumerate}
% 6. Processing
\section{ Processing }
\begin{enumerate}
\item
Verify that the {\tt infile} and {\tt expstatsfile} exist.
%
If {\tt clobber=no}, then verify that the {\tt outfile} does not
exist.
%
Verify that the values of the parameters {\tt numframes}, {\tt
maxframegap}, {\tt minnumframes} and {\tt deltoclkthresh} are in the
valid ranges for these parameters.
%
Note that {\tt numframes} must be an odd number.
\item
If the data are being reprocessed by a user, then perform the following
two steps.
%
\begin{enumerate}
\item
Set {\tt STATUS} bit 10 (of 0-31) to zero (i.e.\ unset it) for all
of the events in the {\tt outfile}. Subsequent processing sets this
bit to one for the appropriate events.
\item
Remove the contents of the HDU ``{\tt BADOCLK}'' in the {\tt
expstatsfile}. This HDU is updated appropriately in subsequent
processing.
\end{enumerate}
\item
Perform the following steps separately for each node of each CCD.
%
\begin{enumerate}
\item
For each valid%
%
\footnote{Here, a frame is valid if:
%
\begin{itemize}
\item[]
the value of {\tt EXPNO} for the frame is greater than the value
of {\tt EXPNO} for the preceding frame (if any),
\item[]
the value of {\tt EXPNO} for the frame is less than the value of
{\tt EXPNO} for the subsequent frame (if any), and
\item[]
there is {\tt OVRCLOCK} data available for the frame.
\end{itemize}
%
If one or more of these conditions are not satisfied, then the frame
is considered invalid. Invalid frames are not included in the
``{\tt BADOCLK}'' HDU of the {\tt expstatsfile}. Any events that
may occur during invalid frames do not have {\tt STATUS} bit 10 set
to one.}
%
frame $k$, compute the expected value $M_{k}$ of {\tt
OVRCLOCK}. In most cases, $M_{k}$ is the median%
%
\footnote{If the number of elements in the set for which the median
is being computed is odd, then the median is the middle value of the
set after the set has been sorted (e.g.\ the median of
[1,1,1,4,5,6,7] is 4). If the number of elements is even, then the
median is the mean of the middle two values of the sorted set. If
the mean is not an integer, then the mean is truncated to obtain an
integer (e.g.\ the median of [1,1,1,4,5,6] is 2, not 2.5).}
%
of the values of {\tt OVRCLOCK} for the frames from $k-n$ to $k+n$,
inclusive, where
%
\begin{equation}
n = \frac{{\tt numframes} - 1}{2}.
\end{equation}
%
There are some cases that require special treatment.
%
\begin{enumerate}
\item
If one or more of the frames immediately before and/or
immediately after frame $k$ are invalid, then use the nearest
valid frames, not the invalid frames to compute the median. For
example, if ${\tt numframes} = 5$, then the {\tt OVRCLOCK} data
for the consecutive valid frames $i$, $j$, $k$, $l$, and $m$ are
used to compute the median, provided that ${\tt EXPNO}_{i} <
{\tt EXPNO}_{j} < {\tt EXPNO}_{k} < {\tt EXPNO}_{l} < {\tt
EXPNO}_{m}$ and that
%
\begin{eqnarray}
{\tt EXPNO}_{j} - {\tt EXPNO}_{i} & \le & {\tt maxframegap},
\label{eqn01} \\
{\tt EXPNO}_{k} - {\tt EXPNO}_{j} & \le & {\tt maxframegap},
\label{eqn02} \\
{\tt EXPNO}_{l} - {\tt EXPNO}_{k} & \le & {\tt maxframegap},\
{\rm and} \label{eqn03} \\
{\tt EXPNO}_{m} - {\tt EXPNO}_{l} & \le & {\tt maxframegap}.
\label{eqn04}
\end{eqnarray}
%
If equation~\ref{eqn01} is not satisfied, then exclude the data
for frame ${i}$ from the computation of the median. If
equation~\ref{eqn02} is not satisfied, then exclude the data for
frames $i$ and $j$. Similarly, if equation~\ref{eqn03} is not
satisfied, then exclude the data for frames $l$ and $m$ and if
equation~\ref{eqn04} is not satisfied, then exclude the data for
frame $m$. {\red That is, in no circumstances should the data
set used to compute the median extend across a gap in frame
numbers that is larger than {\tt maxframegap}.}
\item
At or near the beginning of the data, there may not be $n$ valid
frames preceding frame $k$. For example, for the first valid
frame, there are no valid frames that precede it. In this case,
the median is computed using the data for the first frame and
for the $n$ subsequent, valid frames (provided that the
subsequent frames do not violate the {\tt maxframegap}
constraint). For the second valid frame, the median is computed
using the data for the first and second valid frames and for the
$n$ subsequent, valid frames. (Again, the {\tt maxframegap}
constraint applies.)
\item
Likewise, at or near the end of the data, there may not be $n$
valid frames following frame $k$.
\item
There must be at least ${\tt minnumframes}$ valid frames in the
set of data used to compute $M_{k}$. If there are fewer than
${\tt minnumframes}$ valid frames, due to violations of the {\tt
maxframegap} constraint and/or due to the lack of data at the
beginning or the end of a data set, then set $M_{k}$ equal to
the value of {\tt OVRCLOCK} for frame $k$ (i.e.\ there is too
little data to determine whether the value
of {\tt OVRCLOCK} is anomalous).%
%
\footnote{In this case, frame $k$ is not included in the ``{\tt
BADOCLK}'' HDU of the {\tt expstatsfile} and any events that
occur during frame $k$ do not have {\tt STATUS} bit 10 set to
one.}
%
A consequence of this condition is that an {\tt infile} must
have at least {\tt minnumframes} valid frames of data for a node
in order to search for anomalous values of {\tt OVRCLOCK} for
the node.
\end{enumerate}
\item
If the value of {\tt OVRCLOCK} for a valid frame is greater than or
equal to $M_{k} + {\tt deltoclkthresh}$ or is less than or equal to
$M_{k} - {\tt deltoclkthresh}$, then the value of {\tt OVRCLOCK} is
anomalous. In this case,
%
\begin{enumerate}
\item
modify the {\tt outfile} to
set {\tt STATUS} bit 10 (of 0-31) to one for all of the events
during frame $k$ for the particular CCD and node in question and
\item
update the ``{\tt BADOCLK}'' HDU of the {\tt expstatsfile} to
include the {\tt EXPNO}, {\tt CCD\_ID}, and {\tt NODE\_ID}
information for the frame, CCD, and node in question.
\end{enumerate}
\end{enumerate}
\end{enumerate}
% 7. TBD
\section{ TBD }
\begin{enumerate}
\item
Recreate Figures \ref{fig03}--\ref{fig05} (especially \ref{fig05}) to
determine what the default value of {\tt maxframegap} should be.
\item
Plot the number of anomalies as a function of {\tt TIME} and {\tt CCD\_ID}
and {\tt NODE\_ID}.
\end{enumerate}
% 8. Figures
\newpage
% 8.1. Fig. 1
\begin{figure}
\begin{center}
\epsfig{file=plot_deltoclk_examples,angle=-90,width=6.5in}
\end{center}
\caption{%
\red Three examples of the relative values of {\tt DELTOCLK} as a
function of frame number. The data are from exposure-statistics files
and the values of {\tt DELTOCLK} have had arbitrary offsets applied to
improve the visual clarity. The black, red, and green curves are the
data for $({\tt OBS\_ID}, {\tt CCD\_ID}, {\tt NODE\_ID}) = (1886,7,0)$,
(11793,9,1), and (13019,9,2), respectively. Anomalous ``spikes'' are
evident in the red and green curves. The black curve has no such
anomalies.
%
\label{fig01}
}
\end{figure}
% 8.2. Fig. 2
\begin{figure}
\begin{center}
\epsfig{file=plot_expno_diffs,angle=0,width=6.5in}
\end{center}
\caption{%
A histogram of the differences between consecutive frame numbers for
63,778 exposure-records files. A separate histogram was computed for
each node of each file. The histogram in this figure is a sum of each
of these histograms. If a difference is greater than 1000, then the
difference is set to 1000. If a difference is less than 0, then the
difference is set to 0. The inset histogram is a plot of the same data,
except that it includes only the data in the range from 0 to 100. The
histogram includes all frames, whether they are valid or not.
%
\label{fig02}
}
\end{figure}
% 8.3. Fig. 3
\begin{figure}
\begin{center}
\epsfig{file=plot_deltoclk_diffsA,angle=-90,width=6.5in}
\end{center}
\caption{%
A histogram of the differences between the values of {\tt DELTOCLK} and
the values of the median {\tt DELTOCLK} for 63,778 exposure-records
files. A separate histogram was computed for each node of each file.
The histogram in this figure is a sum of each of these histograms. The
values of the median were computed using a ${\tt numframes} = 5$, ${\tt
maxframegap} = \infty$, and ${\tt minnumframes} = 3$. If a value of
{\tt DELTOCLK} differs by more than 4096 adu from the corresponding
median, then the difference is set to 4096 or $-4096$ as appropriate.
%
\label{fig03}
}
\end{figure}
% 8.4. Fig. 4
\begin{figure}
\begin{center}
\epsfig{file=plot_deltoclk_diffsB,angle=-90,width=6.5in}
\end{center}
\caption{%
This figure is identical to Figure~\ref{fig02}, except that it includes
only the data in the range from -100 to 100 adu.
%
\label{fig04}
}
\end{figure}
% 8.5. Fig. 5
\begin{figure}
\begin{center}
\epsfig{file=plot_anom_frac,angle=0,width=6.5in}
\end{center}
\caption{%
A plot of the fraction of the values of {\tt DELTOCLK} that are
identified as anomalous as a function of the parameter {\tt
deltoclkthresh}. This plot was produced using the data that is shown in
Figure~\ref{fig02}. The inset histogram is a plot of the same data,
except that it includes only the data in the range from 0 to 20.
%
\label{fig05}
}
\end{figure}
% 9. Finish
\end{document}