\documentstyle[jair,twoside,11pt,theapa,lingmacros,epsf]{article}
\input epsf %% at top of file \input{psfig}

\jairheading{16}{2002}{293-319}{11/01}{5/02}
\ShortHeadings{Predicting Problematic Dialogues}
{Walker, Langkilde-Geary, Wright Hastie, Wright \& Gorin}
\firstpageno{293}

\begin{document}


\title{Automatically Training a Problematic Dialogue Predictor for a Spoken Dialogue System}
\author{\name Marilyn A. Walker \email walker@research.att.com \\
	\name Irene Langkilde-Geary \email ilangkil@isi.edu \\
	\name Helen Wright Hastie  \email hhastie@research.att.com \\
	\name Jerry Wright \email jwright@research.att.com \\
	\name Allen Gorin \email algor@research.att.com \\
\addr AT\&T Shannon Laboratory \\
180 Park Ave., Bldg 103, Room E103 \\
Florham Park, NJ 07932
}

\maketitle

\begin{abstract}

Spoken dialogue systems promise efficient and natural access to a
large variety of information sources and services from any phone.
However, current spoken dialogue systems are deficient in their
strategies for preventing, identifying and repairing problems that
arise in the conversation. This paper reports results on automatically
training a Problematic Dialogue Predictor to predict problematic
human-computer dialogues using a corpus of 4692 dialogues collected
with the {\it How May I Help You}$^{SM}$ spoken dialogue
system.  The Problematic Dialogue Predictor can be immediately applied
to the system's decision of whether to transfer the call to a human
customer care agent, or be used as a cue to the system's dialogue
manager to modify its behavior to repair problems, and even perhaps,
to prevent them. We show that a Problematic Dialogue Predictor using
automatically-obtainable features from the first two exchanges in the
dialogue can predict problematic dialogues 13.2\% more accurately than
the baseline.
\end{abstract}


\section{Introduction}

Spoken dialogue systems promise efficient and natural access to a
large variety of information sources and services from any phone.
Systems that support short utterances to select a particular function
(through a statement such as ``Say credit card, collect or
person-to-person") are saving companies millions of dollars per year. Deployed
systems and research prototypes exist for applications such as
personal email and calendars, travel and restaurant information, and
personal banking
\cite{Baggia98,WFN98,Seneff95,Sanderman98,ChuCarpenter99} {\it inter
alia}. Yet there are still many research challenges: current systems
are limited in the interaction they support and brittle in many
respects.

This paper investigates methods by which spoken dialogue systems can
{\it learn} to support more natural interaction on the basis of their
previous experience. One way that current spoken dialogue systems are
quite limited is in their strategies for detecting and repairing
problems that arise in conversation, such as misunderstandings due to
speech recognition error or misinterpretation. If a problem can be
detected, the system can either transfer the call to a human customer
care agent or modify its dialogue strategy in an attempt to repair the
problem.  We can train systems to improve their ability to detect
problems by exploiting dialogues collected in interactions with human
users where the initial segments of these dialogues are used to train
a Problematic Dialogue Predictor (PDP) to {\it predict} that a problem
is likely to occur.  The output of the PDP can be immediately applied
to the system's decision of whether to transfer the call to a human
customer care agent, or it could potentially be used as a cue to the
system's Dialogue Manager to modify its behavior to repair problems,
and even perhaps, to prevent them.



In previous work, we reported initial results for training a PDP using
a variety of different feature sets \cite{LWRGL99,WLWGL00}. When
analyzing the performance of the fully automatic feature set, we
examined which hand-labelled features made large performance
improvements, under the assumption that future work should focus on
developing automatic features that approximate the information
provided by these hand-labelled features. The analysis indicated that
the hand-labelled {\it SLU-success} feature, which encodes whether the
spoken language understanding ({\sc slu}) component captured the
meaning of each exchange correctly. When this hand-labelled feature
is added to the automatic features, it improved the performance of the
PDP by almost 7.6\%.  This finding led us to develop an {\it
SLU-success} predictor \cite{WLW-icml00} and a new version of the PDP
that we report on here. The new version of the PDP takes as input a
fully automatic version of the {\it SLU-success} feature, which we
call {\it auto-SLU-success}.

We train and test both the  {\it auto-SLU-success} predictor and the PDP on a corpus
of 4692 dialogues collected in an experimental trial of AT\&T's {\it
How May I Help You} ({\sc hmihy}$^{SM}$) spoken dialogue system
\cite{GRW97,AbellaGorin,RG,AGA99}.  In this trial, the {\sc hmihy}  system was installed
at an AT\&T customer care center. {\sc hmihy}  answered calls from live
customer traffic and successfully automated a large number of customer
requests. An example dialogue that {\sc hmihy} completed successfully
is shown in Figure \ref{TSuccessDial}. The phone numbers,
card numbers, and pin numbers in the sample dialogues are artificial.

%%
%50097,1,-4,fem_greeting1,I need to place a call with my calling card please, [ .nspn ] [ .lps ] I need to [ uh ] put a call on my calling card please [ .brth ],calling_card,CALLING_CARD,
%50097,2,-3,card_act,eight four four one zero zero five three zero nine one nine two, [ .brth ] 8 4 4 [ .brth ] 1 6 0z [ .bksp ] 0z 5 3 0z [ .brth ] 9 1 9 2 [ .nspn ] [ .bksp ], card_number,CALLING_CARD,
%50097,3,-2,collect_no,one four seven six four five seven eight three,8 1 4 [ .nspn ] 7 6 4 [ .brth ] 5 7 8 3 [ .nspn ] [ .brth ], phone_number,CALLING_CARD,
%50097,4,-1,dialforme_fwdno_rep,eight one four seven six four five seven eight three, [ .brth ] [ .nspn ] [ .clrt ] 8 1 4 7 6 4 5 7 8 3 [ .nspn ], phone_number,CALLING_CARD,
%50097,5,0,thank_you,recog,tscript,LabSel,SLUSel, by_call_info
%% 
\begin{figure}[h]
{\small
\begin{center}
\begin{tabular}{|p{.25in}p{4.25in}|} \hline
S1: &  AT\&T How may I help you?           \\
U1: &  I need to [ uh ] put a call on my calling card please  \\
S2: &  May I have your card number, please? \\
U2: &  7 6 5 4 3 2 1 0 9 8  7 6 5 4   \\
S3: &  What number would you like to call? \\
U3: &  8 1 4  7 7 7  6 6 6 6 ({\it misunderstood}) \\
S4: &  May I have that number again?\\
U4: &  8 1 4  7 7 7  6 6 6 6 \\
S5: & Thank you. \\ \hline
\end{tabular}
\end{center}
\vspace{-.2in}
\caption{\label{TSuccessDial} Sample {\sc tasksuccess}  Dialogue}
}
\end{figure}


Note that the system's utterance in S4 consists of a repair
initiation, motivated by the system's ability to detect that the
user's utterance U3 was likely to have been misunderstood. The goal of
the {\it auto-SLU-success} predictor is to improve the system's
ability to detect such misunderstandings.  The dialogues that have the
desired outcome, in which {\sc hmihy} successfully automates the
customer's call, are referred to as the {\sc tasksuccess} dialogues.
Dialogues in which the {\sc hmihy} system did not successfully
complete the caller's task are referred to as {\sc problematic}. These
are described in further detail below.


This paper reports results from experiments that test whether
it is possible to learn to automatically predict that a dialogue will
be problematic on the basis of information the system has: (1) early
in the dialogue; and (2) in real time.  We train an automatic
classifier for predicting problematic dialogues from features that can
be automatically extracted from the {\sc hmihy} corpus. As described above,
one of these features is the output of the {\it auto-SLU-success} predictor, the
{\it auto-SLU-success} feature, which predicts whether or not the
current utterance was correctly understood \cite{WLW-icml00}. The
results show that it is possible to {\it predict} problematic dialogues using
fully automatic features with an
accuracy ranging from 69.6\% to 80.3\%, depending on whether the system
has seen one or two exchanges. It is possible to {\it identify}
problematic dialogues with an accuracy up to 87\%.


Section \ref{exp-sys-data-sec} describes {\sc hmihy} and the dialogue
corpus that the experiments are based on.  Section \ref{ripper-sec}
discusses the type of machine learning algorithm adopted, namely {\sc
ripper} and gives a description of the experimental design. Section
\ref{section:features} gives a breakdown of the features used in these
experiments.  Section \ref{slu-classifier} presents the method of
predicting the feature {\it auto-SLU-success} and gives accuracy
results.  Section \ref{PDP} presents methods used for utilizing {\sc
ripper} to train the automatic Problematic Dialogue Predictor and
gives the results.  We delay our discussion of related work to Section
\ref{rel-work-sec} when we can compare it to our approach.  Section
\ref{fut-sec} summarizes the paper and describes future work.

%{\bf Classes of Dialogues in HMIHY} \\
%\begin{itemize}
%\item {\bf Dead Line}: no user utterances in  transcriptions \\  ({\it 1585 dialogues})
%\item {\bf Successful}:
%	\begin{itemize}
%	\item {\bf TSuccess}: The system completed the task successfully \\ ({\it 3058 dialogues})
%	\end{itemize}
%\item {\bf Problematic}:
%	\begin{itemize}
%	\item {\bf SysGaveUp}: The system bailed out to an operator  \\  ({\it 1385 dialogues})
%	\item {\bf UserHangUp}: The user hung up on the system  \\  ({\it 707 dialogues})
%	\item {\bf Wizard}: The human operator took over from the system   \\  ({\it  516 dialogues})
%	\item {\bf TFailure}: The system believed that it succeeded but it actually failed to complete the task successfully  \\   %({\it 493 dialogues})
%	\end{itemize}
%\end{itemize}

\section{The HMIHY Data}
\label{exp-sys-data-sec}


%5 This para paraphrased from Arai, Wright, Riccardi and Gorin.

{\sc hmihy} is a spoken dialogue system based on the notion of {\it call
routing} \cite{GRW97,ChuCarpenter99}. 
In the {\sc hmihy} call routing
system, services that the user can access are classified into 14
categories, plus a category called {\it other} for tasks that are not
covered by the automated system and must be transferred to a human
operator \cite{GRW97}. Each category describes a different task, such
as person-to-person dialing, or receiving credit for a misdialed
number.  The system determines which task the caller is requesting on
the basis of its understanding of the caller's response to the
open-ended system greeting {\it AT\&T, How May I Help You?}.  Once the
task has been determined, the information needed for completing the
caller's request is obtained using dialogue submodules that are
specific for each task \cite{AbellaGorin}.

The {\sc hmihy} system consists of an automatic speech recognizer, a
spoken language understanding module, a dialogue manager, and a
computer telephony platform. During the trial, the behaviors of all
the system modules were automatically recorded in a log file, and
later the dialogues were transcribed by humans and labelled with one
or more of the 15 task categories, representing the task that the
caller was asking {\sc hmihy} to perform, on a per utterance basis.
The log files also included labels indicating whether the wizard had
taken over the call or the user had hung up. Our experiments use the
log files to extract automatically obtainable features used as
predictors, and to define the classes of dialogues that we want to
learn to predict. The corpus of 4692 dialogues used in our experiments was
collected in several experimental trials of {\sc hmihy} on live customer
traffic \cite{RG,AGA99}, and is referred to as HM2 in \cite{RG}.
The dialogues vary in length, 97\% are five exchanges or less with
23\% of all the dialogues consisting of only two exchanges.


As mentioned above, dialogues in which {\sc hmihy} successfully
automates the customer's call, as illustrated in Figure 1, are
referred to as {\sc tasksuccess}. Other calls, which are problematic,
are divided into three categories. The first category, referred to as
{\sc hangup}, results from a customer's decision to hang up on the
system.  A sample {\sc hangup} dialogue is in Figure
\ref{HangupDial}. A caller may hang up because s/he is frustrated with
the system; our goal is to learn from the corpus which system
behaviors led to the caller's frustration.

%
%60124,1,-3,fem_greeting1,I need credit please, [ .epr you ] [ .brth ] I need credit please [ .brth ],billing_credit,BILLING_CREDIT,
%60124,2,-2,credit_menu,I mean yes I'd like you get in japan area code and I needed five five, [ .brth ] [ um ] [ .lps ] miss [ uh ] [ uh ] miss in different area code than I needed [ uh ] [ uh ] [ uh ] [ .bksp ],billing_credit,BILLING_CREDIT,
%60124,3,-1,credit_menu_rep,no it is a wrong number, [ oh ] it is a wrong number,billing_credit,BILLING_CREDIT,
%60124,4,0,credit_get_no,recog,tscript,LabSel,SLUSel, by_user (hangup) by_call_info
%%
\begin{figure}[htb!]
%\hline
%
{\small
\begin{center}
\begin{tabular}{|p{.25in}p{4.25in}|} \hline
S1: & AT\&T How may I help you?           \\
U1: & I need credit please. \\
S2: & Okay. What is the reason for the credit? \\
U2: & Miss, uh, Miss, different area code than I needed. \\
S3: & Sorry. I need to know whether you need credit for a wrong number, bad connection or a call that was cut off. \\
U3: & It is a wrong number. ({\it impatient humming}) \\
S4: & What was the number that you dialed? \\
    & USER HANGS UP \\ \hline
\end{tabular}
\end{center}
\vspace{-.2in}
\caption{\label{HangupDial} Sample {\sc hangup} Dialogue}
}
%
%\hline
\end{figure}

The second problematic category ({\sc wizard}), results from a human
customer care agent's decision to take over the call from the
system. Because {\sc hmihy} is experimental, each call during the field
trial was monitored by a human agent serving as a wizard who could
override the system. There were a number of agents who participated as
wizards during the trial of {\sc hmihy} and each wizard was simply told to
take over the call if s/he perceived problems with the system's
performance. The wizard's decision was logged by the experimental
setup, resulting in labelling the call as one that the wizard took
over. Of course we can only infer what might have motivated
the wizard to take over the call, but we assume that the wizard had
good reason for doing so. A dialogue where the wizard decided that
the dialogue was problematic and took over the call is shown in Figure
\ref{WizardDial}.

%
%61496,1,-4,fem_greeting1,recog,tscript,no_information,init,
%61496,2,-3,top_reject_rep,can charge no one eight hundred call A T T, [ .epr ] I'm trying to call [ uh ] 1 8 hundred call A T T [ .nspn ],no_information digit_string,DIAL_FOR_ME,
%61496,3,-2,con_dialforme,hi I'm calling card, [ .epr ] [ .nspn ] [ uh ] through my calling card [ .nspn ],calling_card,CALLING_CARD,
%61496,4,-1,card_act,I need to nine six four nine seven three two two two and then my pin number is four one two nine, [ .nspn ] [ um ] it's 9 6 4 [ .nspn ] 9 7 3 2 [ .nspn ] 2 2 and then my pin number is 4 1 2 3 [ .nspn ],calling_card digit_string,CALLING_CARD,
%61496,5,0,card_rep_tt,recog,tscript,LabSel,SLUSel, by_wizard by_call_info
%%
\begin{figure}[ht!]
{\small
\begin{center}
\begin{tabular}{|p{.25in}p{4.25in}|} \hline
S1: & AT\&T How may I help you?           \\
U1: & ({\it silence}) \\
S2: & Sorry. Please briefly tell me how I may help you? \\
U2: & I'm trying to call 1 8 hundred call A T T.  \\
S3: & Do you want to place a call?  \\
U3: & Through my calling card.  \\
S4: & May I have your card number, please?  \\
U4: & It's 8 7 6  5 4 3 2  1 0 and then my pin number is 8 7 6 5. ({\it misrecognized})  \\
S5: & Sorry. Please enter or say your card number again.  \\
    & WIZARD STEPS IN \\ \hline
\end{tabular}
\end{center}
\vspace{-.2in}
\caption{\label{WizardDial} Sample {\sc wizard}  Dialogue}
}
\end{figure}


The third problematic category, the {\sc taskfailure}
dialogues, are cases where the system completed the call, but carried
out a task that was not the one that the customer was actually
requesting. An example {\sc taskfailure} dialogue is given in Figure
\ref{TFailureDial}: {\sc hmihy} interpreted utterance U2 as a request to
make a third-party call e.g. to {\it bill it to my home phone}. {\sc hmihy}
then asked the caller for the information it needed to carry out this
task, the caller complied, and the system completed the call.

%52320,1,-5,fem_greeting1,recog, [ .nspn ],no_information,init,
%52320,2,-4,top_reject_rep,hello I have it to my home phone, [ .epr you ] I wanta find out how much my bill's worth,att_service,THIRD_NUMBER,
%52320,3,-3,third_act,thanks number, [ um ] the [ access ] number,att_service,THIRD_NUMBER,
%52320,4,-2,collect_no_rep,one eight hundred four four five six zero six three, [ .epr ] [ um ] 1 8 hundred 4 4 5 6 0z 6 3 [ .nspn ], phone_number,DIAL_FOR_ME,
%52320,5,-1,collect_no,DTMF< 800 445 6063 >DTMF,tscript, phone_number,DIAL_FOR_ME,
%52320,6,0,thank_you,recog,tscript,LabSel,SLUSel, by_call_info

\begin{figure}[htb!]
{\small
\begin{center}
\begin{tabular}{|p{.25in}p{4.25in}|} \hline
S1: & AT\&T How may I help you?           \\
U1: & ({\it silence}) \\
S2: & Sorry. Please briefly tell me how I may help you? \\
U2: & I wanta find out how much my bill's worth.  \\
S3: & What phone number would you like the call billed to?  \\
U3: & {\it Um} the access number.  \\
S4: & Sorry. Could I have that phone number again?  \\
U4: & 1 8 hundred 2 2 5 5 2 8 8.  \\
S5: & What number would you like to call? \\
U5: & 800 225 5288 ({\it touchtone input})  \\
S6: & Thank you.\\ \hline
\end{tabular}
\end{center}
\vspace{-.2in}
\caption{\label{TFailureDial} Sample {\sc taskfailure} Dialogue}
}\end{figure}






\section{Experimental Design}
\label{ripper-sec}
\label{exp-design}


Our experiments apply the machine learning program {\sc ripper}
~\cite{CohenML95,CohenAAAI96} to automatically classify the dialogues
as problematic or successful.  {\sc ripper} is a fast and efficient
rule learning system described in more detail in
~\cite{CohenML95,CohenAAAI96}; we describe it briefly here for
completeness.  {\sc ripper} is based on the {\it incremental reduced
error pruning} (IREP) algorithm described in
\cite{FurnkranzWidmer94}. {\sc ripper} improves on IREP with an
information gain metric to guide rule pruning and a Minimum
Description Length or MDL-based heuristic for determining how many
rules should be learned (see Cohen 1995, 1996
%\citepunct{CohenML95,CohenAAAI96} 
for more details). Like other learners, {\sc ripper} takes as input the names
of a set of {\it classes} to be learned, the names and ranges of
values of a fixed set of {\it features}, and {\it training data}
specifying the class and feature values for each example in a training
set. Its output is a {\it classification model} for predicting the
class of future examples, expressed as an ordered set of if-then
rules.

Although any one of a number of learners could be applied to this
problem, we had a number of reasons for choosing {\sc ripper}. First,
it was important to be able to integrate the results of applying the
learner back into the {\sc hmihy} spoken dialogue system. Previous
work suggests that the if-then rules that {\sc ripper} uses to express
the learned classification model are easy for people to understand
\cite{Catlett91,CohenML95}, making it easier to integrate the learned
rules into the {\sc hmihy} system.  Second, {\sc ripper} supports
continuous, symbolic and textual bag (set) features
\cite{CohenAAAI96}, while other learners, such as Classification and
Regression trees (CART) \cite{Brieman}, do not support textual bag
features.  There are several textual features in this dataset that
prove useful in classifying the dialogues.  One of the features that
we wished to use was the string representing the recognizer's
hypothesis.  This is supported in {\sc ripper} because there is no
{\it a priori} limitation on the size of the set. The usefulness of the
textual features is exemplified in Section \ref{results:rules}.
Finally, previous work in which we had applied other learners to the
{\it auto-SLU-success} predictor, utilizing the best performing
feature set with the textual bag features removed, suggested that we
could not expect any significant performance improvements from using
other learners \cite{WLW-icml00}.



%\section{Experimental Design}

In order to train the problematic dialogue predictor (PDP), {\sc
ripper} uses a set of features.  As discussed above, initial
experiments showed that the hand-labelled {\it SLU-success} feature,
which encodes whether an utterance has been misunderstood or not, is
highly discriminatory in identifying problematic dialogues. However,
all the features used to train the PDP must be totally automatic if we
are to use the PDP in a working spoken dialogue system. In order to
improve the performance of the fully automatic PDP, we developed a
fully automatic approximation of the hand-labelled feature, which we
call the {\it auto-SLU-success} feature, in separate experiments with
{\sc ripper}.  The training of the {\it auto-SLU-success} feature is
discussed in Section \ref{slu-classifier}.


Evidence from previous trials of {\sc hmihy} suggest that it is
important to identify problems within a couple of exchanges and 97\%
of the dialogues in the corpus are five exchanges or less.  Thus
features for the first two exchanges are encoded since the goal is to
{\it predict} failures before they happen.  
The experimental architecture of the PDP is illustrated in Figure
\ref{figure:architecture}. This shows how {\sc ripper} is used
first to predict {\it auto-SLU-success} for the first and second
exchanges. This feature is fed into the PDP along with the other
automatic features. The output of the PDP determines whether the
system continues, or if a problem is predicted, the Dialogue Manager
may adapt its dialogue strategy or transfer the customer to a customer
agent.


Since 23\% of the dialogues consisted of only two
exchanges, we exclude the second exchange features for those dialogues
where the second exchange consists only of the system playing a
closing prompt.  We also excluded any features that indicated to the
classifier that the second exchange was the last exchange in the
dialogue.  We compare results for {\it predicting} problematic
dialogues, with results for {\it identifying} problematic dialogues,
when the classifier has access to features representing the whole
dialogue.





\begin{figure}[htb]
\small
\normalsize
\centering
\leavevmode
\epsfxsize = 12 cm
\epsffile{architecture3.eps}
\caption{System architecture using features from the first 2 exchanges\label{figure:architecture}}
\end{figure}

In order to test the {\it auto-SLU-success} predictor as input to the
PDP, we first defined a training and test set for the combined
problem. The test set for the {\it auto-SLU-success} predictor
contains the exchanges that occur in the dialogues of the PDP test
set. We selected a random 867 dialogues as the test set and then
extracted the corresponding exchanges (3829 exchanges). Similarly for
training, the PDP training set contains 3825 dialogues which corresponds
to a total of 16901 exchanges for training the {\it auto-SLU-success}
predictor.

The feature {\it auto-SLU-success} is predicted for each utterance in
the test set, thus enabling the system to be used on new data without
the need for hand-labelling. However, there are two possibilities for
the origin of this feature in the training set.  The first possibility
is for the training set to also consist of solely automatic
features. This method has the potential advantage that the trained
PDP will compensate, if necessary, for whatever noise exists in the {\it
auto-SLU-success} predictions \cite{wright_phd}.  An alternative to
training the PDP on the automatically derived {\it auto-SLU-success}
feature is to train it on the hand-labelled {\it SLU-success} while
still testing it on the automatic feature. This second method is
referred to as ``hand-labelled-training'' or {\it
hlt-SLU-success}. This may provide a more accurate model but it may
not capture the characteristics of the automatic feature in the test
set. Results for these two methods are presented in Section \ref{results:hlt}.


The problem with using {\it auto-SLU-success} for training the PDP is
that the same data is used to train the {\it auto-SLU-success}
predictor. Therefore, we used a cross-validation technique (also
known as jack-knifing) \cite{KulikowskiBook90}, whereby
the training set is partitioned into 4 sets. Three of these sets are
used for training and the fourth for testing. The results for the
fourth set are noted and the process is repeated, rotating the sets
from training to testing. This results in a complete list of predicted
{\it auto-SLU-success} for the training set. The features for the test
set exchanges are derived by training {\sc ripper} on the whole
training set. This process is illustrated in Figure \ref{fig:cross}.


\begin{figure}
\small
\normalsize
\centering
\leavevmode
\epsfxsize = 14 cm
\epsffile{cross.eps}
\caption{Data for segmentation using cross-validation \label{fig:cross}}
\end{figure}





%Section \ref{slu-classifier} presents the method of
%predicting the feature {\it auto-SLU-success} and gives accuracy
%results.  Section \ref{PDP} presents methods used for utilizing {\sc
%ripper} to train the automatic Problematic Dialogue Predictor and
%gives the results.  

The following section gives a breakdown of the input features.
Section \ref{slu-classifier} describes the training and results of the
{\it auto-SLU-success} predictor and Section \ref{PDP} reports the
accuracy results for the PDP.






\section{The Features}
\label{section:features}
\begin{figure}[tbh]
{\begin{center}
\begin{itemize}
\item {\bf Acoustic/ASR Features}
	\begin{itemize}
	\item recog, recog-numwords, asr-duration, dtmf-flag, rg-modality, rg-grammar, tempo
	\end{itemize}
\item  {\bf SLU Features}
	\begin{itemize}
	\item a confidence measure for each of the 15 possible tasks that the user could be trying to do
	\item salience-coverage, inconsistency, context-shift, top-task, nexttop-task, top-confidence, diff-confidence,
	confpertime, salpertime,  auto-SLU-success 
	\end{itemize}
\item {\bf Dialogue Manager and Discourse History Features}
	\begin{itemize}
	\item sys-label, utt-id, prompt, reprompt, confirmation, subdial 
	\item running tallies: num-utts, num-reprompts, percent-reprompts, num-confirms, percent-confirms, num-subdials, percent-subdials
	\item whole dialogue: dial-duration.
	\end{itemize}
\item {\bf Hand-Labelled Features}
	\begin{itemize}
	\item tscript, human-label, age, gender, user-modality, clean-tscript, cltscript-numwords, SLU-success 
	\end{itemize}
\end{itemize}
\caption{\label{features} Features for spoken dialogues.}
\end{center}}
\end{figure}


A dialogue consists of a sequence of exchanges where each exchange
consists of one turn by the system followed by one turn by the user.
Each dialogue and exchange is encoded using the set of 53 features in
Figure~\ref{features}.  Each feature was either automatically logged
by one of the system modules, hand-labelled by humans, or derived from
raw features.  The hand-labelled features are used to produce a {\sc
topline}, an estimation of how well a classifier could do that had
access to perfect information. To see whether our results can
generalize, we also experiment with using a subset of features that
are task-independent described in detail below.



Features logged by the system are utilized because they are produced
automatically, and thus can be used during runtime to alter the course
of the dialogue.  The system modules for which logging information was
collected were the acoustic processor/automatic speech recognizer
({\sc asr}) \cite{RG}, the spoken language understanding ({\sc slu}) module
\cite{GRW97}, and the Dialogue Manager ({\sc dm}) \cite{AbellaGorin}.  Each
module and the features obtained from it are described below.

{\bf Automatic Speech Recognition}:~The automatic speech recognizer
({\sc asr}) takes as input the caller's speech and produces a
potentially errorful transcription of what it believes the caller
said.  The {\sc asr} features for each exchange include the output of the
speech recognizer ({\it recog}), the number of words in the recognizer
output ({\it recog-numwords}), the duration in seconds of the input to
the recognizer ({\it asr-duration}), a flag for touchtone input ({\it
dtmf-flag}), the input modality expected by the recognizer ({\it
rg-modality}) (one of: none, speech, touchtone, speech+touchtone,
touchtone-card, speech+touchtone-card, touchtone-date,
speech+touchtone-date, or none-final-prompt), and the grammar used by
the recognizer ({\it rg-grammar}) \cite{RG}. We also calculate a
feature called {\it tempo} by dividing the value of the {\it
asr-duration} feature by the {\it recog-numwords} feature.

The motivation for the {\sc asr} features is that any one of them may
reflect recognition performance with a concomitant effect on
spoken language understanding.  For example, other work has found {\it
asr-duration} to be correlated with incorrect recognition
\cite{Hirschbergetal99}. The name of the grammar ({\it rg-grammar})
could also be a predictor of {\sc slu} errors since it is well known
that the larger the grammar is, the more likely an {\sc asr} error is.
In addition, the {\it rg-grammar} feature also encodes expectations
about user utterances at that point in the dialogue, which may
correlate to differences in the ease with which any one recognizer could
correctly understand the user's response.  One motivation for the {\it
tempo} feature is that previous work suggests that users tend to slow
down their speech when the system has misunderstood them
\cite{Levow98,SWP92}; this strategy actually leads to more errors
since the speech recognizer is not trained on this type of speech. The
{\it tempo} feature may also indicate hesitations, pauses, or
interruptions, which could also lead to {\sc asr} errors.  On the
other hand, touchtone input in combination with speech, as encoded by
the feature {\it dtmf-flag}, might increase the likelihood of
understanding the speech: since the touchtone input is unambiguous it
can constrain spoken language understanding.



{\bf Spoken Language Understanding}:~The goal of the spoken
language understanding ({\sc slu}) module is to identify which of the
15 possible tasks the user is attempting and extract from the
utterance any items of information that are relevant to completing
that task, e.g. a phone number is needed for the task {\it dial for
me}.

Fifteen of the features from the {\sc slu} module represent the distribution
for each of the 15 possible tasks of the {\sc slu} module's confidence in
its belief that the user is attempting that task \cite{GRW97}.  We
also include a feature to represent which task has the highest
confidence score ({\it top-task}), and which task has the second
highest confidence score ({\it nexttop-task}), as well as the value of
the highest confidence score ({\it top-confidence}), and the
difference in values between the top and next-to-top confidence scores
({\it diff-confidence}).

Other features represent other aspects of the {\sc slu} processing of
the utterance. The {\it inconsistency} feature is an intra-utterance
measure of semantic diversity, according to a task model of the domain
\cite{AbellaGorin}.  Some task classes occur together quite
naturally within a single statement or request, e.g. the {\it dial for
me} task is compatible with the {\it collect call} task, but is not
compatible with the {\it billing credit} task. The {\it
salience-coverage} feature measures the proportion of the utterance
which is covered by the salient grammar fragments.  This may include
the whole of a phone or card number if it occurs within a fragment.
The {\it context-shift} feature is an inter-utterance measure of the
extent of a shift of context away from the current task focus, caused
by the appearance of salient phrases that are incompatible with it,
according to a task model of the domain.

In addition, similar to the way we calculated the {\it tempo} feature,
we normalize the {\it salience-coverage} and {\it top-confidence}
features by dividing them by {\it asr-duration} to produce the {\it
salpertime} and {\it confpertime} features.  The {\it tempo} and the
{\it confpertime} and {\it salpertime} features are used only for
predicting {\it auto-SLU-success}.

The motivation for these {\sc slu} features is to make use of
information that the {\sc slu} module has as a result of processing the
output of {\sc asr} and the current discourse context. For example,
for utterances that follow the first utterance, the {\sc slu} module
knows what task it believes the caller is trying to complete.  The
{\it context-shift} feature incorporates this knowledge of the
discourse history, with the motivation that if it appears that the
caller has changed her mind, then the {\sc slu} module may have
misunderstood an utterance. 


{\bf Dialogue Manager}:
The function of the Dialogue Manager is to take as input the output of
the {\sc slu} module, decide what task the user is trying to accomplish,
decide what the system will say next, and update the discourse
history \cite{AbellaGorin}. The Dialogue Manager decides
whether it believes there is a single unambiguous task that the user
is trying to accomplish, and how to resolve any ambiguity. 

Features based on information that the Dialogue Manager logged about
its decisions or features representing the ongoing history of the
dialogue might be useful predictors of {\sc slu} errors or task
failure.  Some of the potentially interesting Dialogue Manager events
arise due to low {\sc slu} confidence levels which lead the Dialogue Manager
to {\it reprompt} the user or {\it confirm} its understanding. A
reprompt might be a variant of the same question that was asked
before, or it could include asking the user to choose between two
tasks that have been assigned similar confidences by the {\sc slu}
module. For example, in the dialogue in Figure \ref{HangupDial} the
system utterance in S3 counts as a reprompt because it is a variant of
the question in utterance S2.

The features that we extract from the Dialogue Manager are the
task-type label, {\it sys-label}, whose set of values include a value
to indicate when the system had insufficient information to decide on
a specific task-type, the utterance id within the dialogue ({\it
utt-id}), the name of the prompt played to the user ({\it prompt}),
and whether the type of prompt was a reprompt ({\it reprompt}), a
confirmation ({\it confirm}), or a subdialogue prompt (a superset of
the reprompts and confirmation prompts ({\it subdial})).  The {\it
sys-label} feature is intended to capture the fact that some tasks may
be harder than others. The {\it utt-id} feature is motivated by the
idea that the length of the dialogue may be important, possibly in
combination with other features like {\it sys-label}.  
The different prompt features for
initial prompts, reprompts, confirmation prompts and subdialogue
prompts are motivated by results indicating that reprompts and
confirmation prompts are frustrating for callers and that callers are
likely to hyperarticulate when they have to repeat themselves, which
results in {\sc asr} errors \cite{SWP92,Levow98,WKL00-jair}.

The discourse history features included running tallies for the number
of reprompts ({\it num-reprompts}), number of confirmation prompts
({\it num-confirms}), and number of subdialogue prompts ({\it
num-subdials}), that had been played before the utterance currently
being processed, as well as running percentages ({\it
percent-reprompts, percent-confirms, percent-subdials}).  The use of
running tallies and percentages is based on previous work suggesting that
normalized features are more likely to produce generalized predictors
\cite{LWK99}. A feature available for {\it identifying} problematic
dialogues is {\it dial-duration} that is not available for initial
segments of the dialogue.


{\bf Hand Labelling}:~As mentioned above, the features obtained via
hand-labelling are used to provide a {\sc topline} against which to
compare the performance of the fully automatic features. The
hand-labelled features include human transcripts of each user utterance
({\it tscript}), a set of semantic labels that are closely related to
the system task-type labels ({\it human-label}), age ({\it age}) and
gender ({\it gender}) of the user, the actual modality of the user
utterance ({\it user-modality}) (one of: nothing, speech, touchtone,
speech+touchtone, non-speech), and a cleaned transcript with non-word
noise information removed ({\it clean-tscript}). From these features,
we calculated two derived features. The first was the number of words
in the cleaned transcript ({\it cltscript-numwords}), again on the
assumption that utterance length is strongly correlated with {\sc asr} and
{\sc slu} errors. The second derived feature was based on calculating
whether the {\it human-label} matches the {\it sys-label} from the
Dialogue Manager ({\it SLU-success}). This feature is described in detail in the next section.

In the experiments, the features in Figure~\ref{features}, excluding the
hand-labelled features, are referred to as the {\sc automatic} feature
set.  The experiments test how well misunderstandings can be
identified and whether problematic dialogues can be predicted using
the {\sc automatic} features. We compare the performance of the {\sc
automatic} feature set to the full feature set including the
hand-labelled features and to the performance of the {\sc automatic}
feature set with and without the {\it auto-SLU-success} feature.
Figure \ref{Encoding} gives an example of the encoding of some of the
automatic features for the second exchange of the {\sc wizard}
dialogue in Figure~\ref{WizardDial}. The prefix ``{\it e2-}''
designates the second exchange. We discuss several of the features values
here to ensure that the reader understands the way in which the features
are used. In utterance S2 in Figure 3, the system says {\it Sorry please briefly
tell me how I may help you}. In Figure 8, this is encoded by several features.
The feature {\it e2-prompt} gives the name of that prompt, {\tt top-reject-rep}.
The feature {\it e2-reprompt} specifies that S2 is a reprompt, a second attempt by
the system to elicit a description of the caller's problem. The feature {\it e2-confirm} specifies that S2 is not a confirmation prompt.  The feature {\it e2-subdial} specifies that S2 initiates a subdialogue
and {\it e2-num-subdials} encodes that this is the first subdialogue so far, while
{\it e2-percent-subdials} encodes that out of all the system utterances so far,
50\% of them initiate subdialogues. 

%{\bf Feature Encoding For Wizard Dialogue} \\
%
%61496, -4,fem_greeting1,  0, clean_tscript,0,recog,0,0,0,0,0,no_match,0,no_t_rest,0,no_r_rest,0,not_reprompt,0,0,not_confirm,0,0,not_subdial,0,0,tscript,no_information,init,speech_plus_tt,Greeting_gram,nothing, ?, ?,  3.6,0.000,0.000,0.000,none,none,0,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,-3,top_reject_rep,  0,  Im trying to call 1 8 hundred call A T T,11,can charge no one eight hundred call A T T,10,0.636,0.7,3,4,call 1 8 hundred A T T ,7,Im trying to call ,4,can charge no       ,3,reprompt,1,0.5,not_confirm,0,0,subdial,1,0.5,  epr  Im trying to call  uh  1 8 hundred call A T T  nspn ,no_information digit_string,DIAL_FOR_ME,speech_plus_tt,Reprompt_gram,speech,  A,  M ,  6.68,0.000,0.000,0.000,dial_for_me,none,0.810,0.81, 0, 0, 0, 0, 0, 0.810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,-2,con_dialforme,  0,  through my calling card,4,hi Im calling card,4,0.5,0.5,0,0,calling card ,2,through my ,2,hi Im  ,2,not_reprompt,1,0.333,confirm,1,0.333,subdial,2,0.666,  epr   nspn   uh  through my calling card  nspn ,calling_card,CALLING_CARD,speech,Confirm_gram,speech,  A,  M ,  5.88,0.500,0.000,0.027,calling_card,dial_for_me,0.910,0, 0, 0, 0, 0.910, 0, 0.910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,-1,card_act,  0,  its 9 6 4 9 7 3 2 2 2 and then my pin number is 4 1 2 3,20,I need to nine six four nine seven three two two two and then my pin number is four one two nine,22,0.9,0.818,16,14,9 6 4 9 7 3 2 2 2 and then my pin number is 4 1 2 ,18,its 3 ,2,I need to                   9,4,not_reprompt,1,0.25,not_confirm,1,0.25,not_subdial,2,0.5,  nspn   um  its 9 6 4  nspn  9 7 3 2  nspn  2 2 and then my pin number is 4 1 2 3  nspn ,calling_card digit_string,CALLING_CARD,speech_plus_tt_card,Callingcard_gram,speech,  A,  M ,  17.2,0.364,0.000,0.011,dial_for_me,calling_card,0.810,0.053, 0, 0, 0, 0.757, 0, 0.810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,5, 2, 0.4, 1, 0.2, 3, 0.6, 51.32

\begin{figure}[t!]
\begin{center}
{\scriptsize
\begin{tabular}{|rl||rl|}  \hline
{\it e2-recog}:          &\multicolumn{3}{l|}{can charge no one eight hundred call A T T}   \\ \hline
{\it e2-rg-modality}:&speech-plus-touchtone& {\it e2-recog-numwords}: & 10 \\ \hline 
{\it e2-user-modality}:  & speech   & {\it e2-dtmf-flag}: & 0     \\ \hline
{\it e2-rg-grammar}: &Reprompt-gram & {\it e2-asr-duration}: & 6.68 \\ \hline 
{\it e2-top-task }:& dial-for-me& {\it e2-top-confidence  }:& .81 \\ \hline 
{\it e2-nexttop-task }:& none   & {\it e2-diff-confidence }:& .81 \\ \hline
{\it e2-salience-coverage}: & 0.000 & {\it e2-task1  }:& 0  \\ \hline 
{\it e2-inconsistency}: & 0.000     & {\it e2-task2}:& 0 \\ \hline
{\it e2-context-shift}: & 0.000     & {\it e2-task3}:& 0  \\ \hline 
{\it e2-prompt }: & top-reject-rep  & {\it e2-task4}:& 0 \\\hline 
{\it e2-reprompt}: & reprompt       & {\it e2-task5}:& 0 \\ \hline
{\it e2-num-reprompts}: & 1 	    & {\it e2-task6}:& .81 \\ \hline
{\it e2-percent-reprompts}: & 0.5   & {\it e2-task7}:& 0 \\ \hline
{\it e2-confirm}: & not-confirm     & {\it e2-task8}:& 0 \\ \hline
{\it e2-num-confirms}: & 0	    & {\it e2-task9}:& 0 \\ \hline
{\it e2-percent-confirms}: & 0	    & {\it e2-task10}:& 0 \\ \hline
{\it e2-subdial}: & subdial         & {\it e2-task11}:& 0 \\ \hline
{\it e2-num-subdials}: & 1 	    & {\it e2-task12}:& 0 \\\hline 
{\it e2-percent-subdials}: & 0.5    & {\it e2-task13}:& 0 \\ \hline 
{\it e2-cltscript-numwords}: & 11   & {\it e2-task14}:& 0 \\ \hline
{\it e2-sys-label}: & DIAL-FOR-ME   & {\it e2-task15}:& 0 \\ \hline
{\it e2-human-label}:&no-info  digitstr& {\it e2-no-info}:& 1 \\ \hline 
{\it e2-tscript}:        &\multicolumn{3}{l|}{{\it epr} I'm trying to call {\it uh} 1 8 hundred call A T T {\it nspn}} \\ \hline{\it e2-clean-tscript}:  &\multicolumn{3}{l|}{I'm trying to call 1 8 hundred call A T T}	  \\ \hline
%%  &\multicolumn{3}{l|}{. . .}  \\ \hline
%%{\it num-subdials}:& 3              & {\it num-utts }: & 5    \\ \hline 
%%{\it percent-subdials}:& 0.6        & {\it dial-duration }:& 51.32 \\\hline
\end{tabular}}
\end{center}
\caption{\label{Encoding} Feature encoding for Second Exchange of {\sc wizard }  dialogue.}
\end{figure}

As mentioned earlier, we are also interested in generalizing our
problematic dialogue predictor to other systems.  Thus, we trained
{\sc ripper} using only features that are both automatically
acquirable during runtime and independent of the {\sc hmihy} task.
The subset of features from Figure~\ref{features} that fit this
qualification are in Figure~\ref{ATfeatures}. We refer to them as the
{\sc auto, task-indept} feature set. Examples of features that are not
task-independent include {\it recog-grammar, sys-label, prompt} and
the hand-labelled features.

\begin{figure}[t]
\begin{center}
\begin{itemize}
\item {\bf Acoustic/ASR Features}
	\begin{itemize}
	\item recog, recog-numwords, asr-duration, dtmf-flag, rg-modality actual modality of the user utterance.
	\end{itemize}
\item  {\bf SLU Features}
	\begin{itemize}
	\item  salience coverage, inconsistency, context-shift, top confidence, diff-confidence, auto-SLU-success.
	%% salpertime, confpertime, 
	\end{itemize}
\item {\bf Dialogue Manager Features}
	\begin{itemize}
	\item utterance by utterance: utt-id, reprompt, confirmation, subdial
	\item running tallies: num-utts, num-reprompts, percent-reprompts, num-confirms, percent-confirms, num-subdials, percent-subdials, dial-duration
	\end{itemize}
\end{itemize}
\caption{\label{ATfeatures} Automatic task-independent features available at runtime.}
\end{center}
\end{figure}



\section{Auto-SLU-success Predictor}
\label{slu-classifier}
\label{results1-sec}

The goal of the {\it auto-SLU-success} predictor is to identify, for
each exchange, whether or not the system correctly understood the
user's utterance. As mentioned above, when the dialogues were
transcribed by humans after the data collection was completed, the
human labelers not only transcribed the users' utterances, but also
labelled each utterance with a semantic category representing the task
that the user was asking {\sc hmihy} to perform. This label is called
the {\it human-label}.  The system's Dialogue Manager decides among
several different hypotheses produced by the {\sc slu} module, and logs its
hypothesis about what task the user was asking {\sc hmihy} to perform;
the Dialogue Manager's hypothesis is
known as the {\it sys-label}.  We distinguish four classes of spoken
language understanding outcomes based on comparing the
{\it human-label}, the {\it sys-label} and recognition results for
card and telephone numbers: (1) {\sc rcorrect}: {\sc slu} correctly
identified the task and any digit strings were also correctly
recognized; (2) {\sc rpartial-match}: {\sc slu} correctly recognized
the task but there was an error in recognizing a calling card number
or a phone number; (3) {\sc rmismatch}: {\sc slu} did not correctly
identify the user's task; (4) {\sc no-recog}: the recognizer did not
get any input to process and so the {\sc slu} module did not
either. This can arise either because the user did not say
anything or because the recognizer was not listening when the user
spoke.  The {\sc rcorrect} class accounts for 7481 (36.1\%) of the
exchanges in the corpus.  The {\sc rpartial-match} accounts for 109
(0.5\%) of the exchanges. The {\sc rmismatch} class accounts for 4197
(20.2\%) of the exchanges and the {\sc no-recog} class accounts for
8943 (43.1\%) of the exchanges.


The {\it auto-SLU-success} predictor is trained using 45 fully automatic
features.  These features are the Acoustic/{\sc asr} features, {\sc slu} features and Dialogue Manager and Discourse History features, given in Figure \ref{features} . Hand-labelled features were not used. 





%\subsection{Auto-SLU-success Predictor Results}

We evaluate the four-way {\it auto-SLU-success} classifier by reporting accuracy, precision, recall and the categorization confusion matrix. This classifier is trained on all the features for the whole training set, and then tested on the held-out test set. 

%We also examine the extent to which we can minimize the error on
%the error classes {\sc rmismatch} and {\sc rpartial-match} by
%manipulating {\sc ripper}'s loss ratio parameter.


Table \ref{slu-results-table} summarizes the overall accuracy results of the system trained on the whole training set and tested on the test set described in Section  \ref{exp-design}.
The first line of Table \ref{slu-results-table} represents the
accuracy from always guessing the majority class ({\sc no-recog});
this is the {\sc baseline} against which the other results should be
compared.
%%[HELEN I CHANGED THIS IN THE TABLE, PREVIOUS PAPER HAD 56.9 here ASSUMED WAS AN OVERSIGHT].  
The second row, labelled {\sc automatic}, shows the accuracy based on
using all the features available from the system modules.  This
classifier can identify {\sc slu} errors 47.0\% better than the
baseline. An experiment was run to see if the cross-validation method
described in Section \ref{exp-design} performs worse than using the
whole data on the same test set. This experiment showed that there was little loss of accuracy when using cross-validation (0.6\%).

\begin{table}[htb]
\begin{center}
\vskip 0.15in
\begin{tabular}{|l|c|c|} \hline
{\bf Features Used}  & {\bf Accuracy}    \\ \hline \hline
{\sc baseline} (majority class)  & 43.1\%       \\ \hline % largest class is no-recog
{\sc automatic}     & 90.1 \%  \\ 
\hline 
\end{tabular}
\end{center}
\caption{Results for detecting {\sc slu} Errors using {\sc ripper}}
\label{slu-results-table}
\end{table}


Figure \ref{allfeats-rule-fig} shows some top performing rules that
{\sc ripper} learns when given all the features. These rules directly
reflect the usefulness of the {\sc slu} features. Note that some of
the rules use {\sc asr} features in combination with {\sc slu}
features such as {\it salpertime}.  Previous studies \cite{WLW-icml00}
have also shown {\sc slu} features to be useful.  We had also
hypothesized that features from the Dialogue Manager and the discourse
history might be useful predictors of {\sc slu} errors, however these
features rarely appear in the rules with the exception of {\it
sys-label}. This is in accordance with previous experiments
which show that these features do not add
significantly to the performance of the {\sc slu only} feature set
\cite{WLW-icml00}.

\begin{figure*}[t]
%%\vskip 0.15in
\rule{6in}{.2mm} \\
%\small
%{
{\bf if} (sys-label = DIAL-FOR-ME)  $\wedge$ (dtmf-flag = 0)  $\wedge$ (recog  contains  "from")  $\wedge$ (recog-numwords $\leq$ 8) {\bf then} {\it rpartial-match} \\
{\bf if} (sys-label = DIAL-FOR-ME)  $\wedge$  (dtmf-flag = 0)  $\wedge$  (asr-duration $\geq$ 4.08)  $\wedge$  (recog-grammar = Billmethod-gram)  $\wedge$ (recog  contains  "my")  $\wedge$ (recog-numwords $\leq$ 8) {\bf then} {\it rpartial-match} \\
{\bf if} (spoken-digit = 1)  $\wedge$ (salpertime $\leq$ 0.05) $\wedge$ (top-confidence $\leq$ 0.851) {\bf then} {\it rmismatch} \\
{\bf if} (spoken-digit = 1) $\wedge$  (salpertime $\leq$ 0.05) $\wedge$ (confpertime $\leq$ 0.076)  {\bf then} {\it rmismatch}\\
{\bf if} (spoken-digit = 1)  $\wedge$ (top-confidence $\leq$ 0.836) {\bf then} {\it rmismatch}\\
{\bf if} (spoken-digit = 1) $\wedge$  (salpertime $\leq$ 0.05) $\wedge$  (sys-label = CALLING-CARD) {\bf then} {\it rmismatch}\\
{\bf if} (asr-duration $\geq$ 1.04)  $\wedge$  (utt-id $\geq$ 2)  $\wedge$ (sys-label = DIAL-FOR-ME)  $\wedge$  (diff-confidence $\leq$ 0.75) {\bf then} {\it rmismatch}\\
\rule{6in}{.2mm} \\
\caption{A subset of rules learned by {\it ripper} when given the automatic features for determining {\it auto-SLU-success}}
\label{allfeats-rule-fig}
\vspace{-.2in}
\end{figure*}



%[EXPAND: new feature set experiments??]

We also report precision and recall for each category on the held-out
test set.  The results are shown in Tables \ref{prerec-table} and
\ref{prerec-matrix}.  Table \ref{prerec-table} shows that the
classification accuracy rate is a result of a high rate of correct
classification for the {\sc rcorrect} and {\sc no-recog} class, at the
cost of a lower rate for {\sc rmismatch} and {\sc rpartial-match}.
This is probably due to the fact that there are fewer
examples of these categories in the training set.

%[CHECK]or prior probability ripper swerve


\begin{table}[htb]
\begin{center}
%\vskip 0.15in
\begin{tabular}{|r|c|c|} \hline
{\bf Class}  &{\bf Recall} & {\bf Precision}  \\ \hline 
{\sc rcorrect}    & 92.6\%     &   86.8\%        \\ \hline 
{\sc no-recog}     & 98.5\%     &97.5\%        \\ \hline 
{\sc rmismatch}  &  70.6\%     &  81.0\%        \\ \hline 
{\sc rpartial-match}& 22.7\%&40.0\%\\ \hline
\end{tabular}
\end{center}
%\vspace{-.2mm}
\caption{Precision and Recall for Test set using Automatic features}
\label{prerec-table}
\end{table}




\begin{table}[htb]
\begin{center}
\vskip 0.15in
\begin{tabular}{|r|c|c|c|c|} \hline
 &{\sc rcorrrect} & {\sc no-recog}& {\sc rmismatch } &{\sc rpartial }  \\ \hline 
{\sc rcorrect} &2784&6&211&5 \\ \hline
{\sc no-recog} &  9    &3431 &44        & 0                  \\ \hline 
{\sc rmismatch} & 409    & 83        &1204  &10              \\ \hline 
{\sc rpartial-match}  & 6    & 0   & 28 & 10                \\ \hline 
\end{tabular}
\end{center}
\caption{Confusion Matrix for Test set using Automatic features}
%: Rcor = {\sc rcorrect}, Rmis = {\sc rmismatch}, Rpar = {\sc rpartial-match}}
%%\vspace{-.2mm}
\label{prerec-matrix}
\end{table}

In some situations, one might not need to distinguish between the
different misunderstanding categories: {\sc no-recog, rmismatch} and
{\sc rpartial-match}. Therefore, experiments were performed that collapsed
these 3 problematic categories into one category ({\sc rincorrect}). This
resulted in a recognition accuracy of 92.4\%, a 29.4\% improvement
over the baseline of 63\%, which is the percentage of {\sc rincorrect}
exchanges. The precision and recall matrix is given in Table
\ref{prerec-table-2cat}.


\begin{table}[htb]
\begin{center}
%% HAS NEW NUMBERS.
\vskip 0.15in
\begin{tabular}{|r|c|c|} \hline
{\bf Class}  &{\bf Recall} & {\bf Precision}  \\ \hline 
{\sc rcorrect}    & 91.2\%     & 89.0  \%        \\ \hline 
{\sc rincorrect}  &  93.1\%     &  94.5\%        \\ \hline 
\end{tabular}
\end{center}
\caption{Precision and Recall for Test set using Automatic features}
%\vspace{-.2mm}
\label{prerec-table-2cat}
\end{table}


%[I COMMENTED OUT THE STUFF ON MANIPULATING THE LOSS RATIO HERE]
%[REPORT CROSS VALIDATION RESULTS AS WELL AS HELD OUT, OR JUST HELD OUT?
%Compare preds-output with hand-labelled.]

%We use the output of the SLU-success predictor on the test data as input to the
%Problematic Dialogue Predictor.

% However, for the {\sc hmihy} application, the classification errors
% that are most detrimental to the system's performance are false
% acceptances, i.e. cases where the system has misunderstood the
% utterance but goes ahead with the dialogue without realizing it. This
% situation was illustrated in Table 2, and it resulted in a task
% failure.  Errors where the system has correctly understood but {\it
% reprompts} the user or {\it confirms} its understanding are much less
% detrimental to overall performance.  Thus we carried out a second set
% of experiments in which we parameterized {\sc ripper} to avoid this
% kind of error.  The results for the same training and test set are
% shown in Tables \ref{prerec2-table} and
% \ref{prerec2-matrix}.  This parameterization increases the recall of
% the {\sc rmismatch} class to 82.31\%, at the cost of a decrease in
% recall for the {\sc rcorrect} class.


%ganges[130]% more summarize-predicted-loss-jair-utt
%320 errors in 2198 trials (14.56%)

%recall  precis. fallout F[0.5]  F[1.0]  F[2.0]  focus class
%------  ------- ------- ------  ------- ------- -----------
% 89.17   90.29   18.58  0.9007  0.8973  0.8939  Rcorrect
% 79.26   77.55   11.36  0.7789  0.7840  0.7891  Rmismatch
% 40.00   36.36    0.64  0.3704  0.3810  0.3922  Rpartial_match

% Rcor  Rmis  Rpar     <-classified as
% ----  ----  ----
% 1293   157     0     Rcor: Rcorrect
%  137   577    14     Rmis: Rmismatch
%    2    10     8     Rpar: Rpartial_match     

% \begin{table}[htb]
% \begin{center}
% %% NEW NUMBERS
% \caption{Precision and Recall using the {\sc  automatic} feature set, Minimizing False Acceptance}
% \vskip 0.15in
% \begin{tabular}{|r|cc|} \hline
% {\bf Class}  &{\bf Recall} & {\bf Precision}  \\ \hline 
% Rcorrect     & 89.17 \%     & 90.29  \%        \\ \hline 
% Rmismatch    & 79.26 \%     &  77.55 \%        \\ \hline 
% Rpartial-match  & 40.00 \%     & 36.26 \%        \\ \hline 
% \end{tabular}
% \end{center}
% %%\vspace{-.2mm}
% \label{prerec2-table}
% \end{table}

% \begin{table}[htb]
% \begin{center}
% %% NEW NUMBERS
% \caption{Confusion Matrix for Test set, Minimizing False Acceptance:
% Rcor = Rcorrect, Rmis = Rmismatch, Rpar = Rpartial-match}
% \vskip 0.15in
% \begin{tabular}{|r|c|c|c|} \hline
% KEY  & Rcor &  Rmis &  Rpar   \\ \hline 
%  Rcorrect &1293     &  157        &  0                \\ \hline 
%  Rmismatch &137     &  577        &  14              \\ \hline 
%  Rpartial & 2    &  10     &  8               \\ \hline 
% \end{tabular}
% \end{center}
% %%\vspace{-.2mm}
% \label{prerec2-matrix}
% \end{table}


\section{Problematic Dialogue Predictor}
\label{PDP}

The goal of the PDP is to predict, on the basis of
information that it has early in the dialogue, whether or not the
system will be able to complete the user's task. The output classes
are based on the four dialogue categories described above. However, as 
{\sc hangup, wizard} and {\sc taskfailure} are treated as equivalently
problematic by the system, as illustrated in Figure \ref{figure:architecture}, these 3 categories are collapsed into  {\sc problematic}. Note that this
categorization is inherently noisy because it is impossible to know the real
reasons why a caller hangs up or a wizard takes over the call. The
caller may hang up because she is frustrated with the system, or she
may simply dislike automation, or her child may have started
crying. Similarly, one wizard may have low confidence in the system's
ability to recover from errors and use a conservative approach that
results in taking over many calls, while another wizard may be more
willing to let the system try to recover. Nevertheless, we take these
human actions as a human labelling of these calls as
problematic. Given this binary classification, approximately 33\% of the
calls in the corpus of 4692 dialogues are {\sc problematic} and 67\%
are {\sc tasksuccess}.



\subsection {Problematic Dialogue Predictor Results}
\label{results2-sec}


This section presents results for predicting problematic dialogues.
Taking into account the fact that a problematic dialogue must be predicted at a point in the dialogue where the system can do something about it,
we compare {\it prediction} accuracy after having seen only the first
exchange or the first two exchanges with {\it identification} accuracy after
having seen the whole dialogue.  For each of these situations, we also
compare results for the {\sc automatic} feature set (as described
earlier) with and without the {\it auto-SLU-success} feature and with the hand-labelled feature {\it SLU-success}. 


% We first examine the
% overall accuracy of the various classifiers, and then turn to the
% question of whether in practice the system should be optimizing
% overall accuracy, or maximizing automation or minimizing failed
% automation.


Table \ref{pdp-results-table} summarizes the overall accuracy results.
The three columns present results for Exchange 1, Exchanges 1\&2 and
over the whole dialogue. The first row gives the baseline result which
represents the prediction accuracy from always guessing the majority
class. Since 67.1\% of the dialogues are {\sc tasksuccess} dialogues,
we can achieve 67.1\% accuracy from simply guessing {\sc tasksuccess}
for each dialogue. The second row gives results using only automatic
features, but without the {\it auto-SLU-success} feature. The third
row uses the same automatic features but adds in {\it
auto-SLU-success}. This feature is obtained for both the training and
the test set, using the cross-validation method discussed in Section
\ref{exp-design}. The fourth and fifth rows show results using the subset of
features that are both fully automatic and task-independent as described
in Section \ref{section:features}.

The automatic results given in row 2 are significantly higher by a paired t-test than the baseline for all three sections of the dialogue (df=866, t=2.1, p=0.035;df=866, t=7.2, p=0.001;df=866 t=13.4, p=0.001).

Rows 6 and 7 show accuracy improvements gained by
the addition of hand-labelled features. These rows give a {\sc
topline} against which to compare the results in rows 2, 3, 4 and 5.
Results using all the automatic features plus the hand-labelled {\it
SLU-success} are given in row 6. In these experiments, the
hand-labelled {\it SLU-success} feature is used for training and testing. Comparing this result with the second row shows that if one had a perfect
predictor of {\it auto-SLU-success} in the training and the test set,
then this feature would increase accuracy by 5.5\% for Exchange 1
(from 70.1\% to 75.6\%); by 7.6\% for Exchanges 1\&2 (from 78.1\% to
85.7\%); and by 5.9\% for the whole dialogue (87.0\% to 92.9\%). These
increases are significant by a paired t-test (df=866, t=5.1, p=0.0001; df=866, t=2.1, p=0.035; df=866, t=6.7, p=0.001).

%The difference in performance between the results using {\it auto-SLU-success} in row 2 and using the hand-labelled {\it SLU-success} in row 6 suggests that it might be possible to achieve even better performance if we had a better {\it auto-SLU-success} predictor.


Comparing the result in row 6 with the result in row 3 shows that the
{\it auto-SLU-success} predictor that we have trained can improve
performance, but could possibly help more with different training
methods. Ideally, the result in row 3, for automatic features plus
{\it auto-SLU-success}, should fall between the figures in rows 2 and
6, and be closer to the results in row 6. With Exchanges 1\&2, adding
{\it auto-SLU-success} results in an increase of 1.1\% which is not
significant (compare rows 2 and 3).  For Exchange 1 only, {\sc ripper}
does not use the {\it auto-SLU-success} feature in its ruleset and
does not yield an improvement over the system trained only on the
automatic features. The system trained on the whole dialogue with
automatic features plus {\it auto-SLU-success} also does not yield an
improvement over the system trained without {\it auto-SLU-success}.

\subsubsection{Task-independent Features}
Rows 4 and 5 give the results using the {\sc auto, task-indept}
feature set described in Figure \ref{ATfeatures} without and with the
{\it auto-SLU-success} feature, respectively. These results are
significantly above the baseline using a paired t-test, with Exchanges
1\&2 giving an increase of 13.1\% (df=866, t=8.6, p=0.001) using {\sc
task-indept} features with {\it auto-SLU-success}.  By comparing rows
4 and 5, one observes an increase in the {\sc auto, task-indept}
features set when the feature {\it auto-SLU-success} is added using
Exchanges 1\&2 and whole dialogue. The 1.9\% increase for Exchanges
1\&2 shows a trend (df=866, t=1.7,p=0.074), whereas the 2\% increase
for the whole dialogue is statistically significant by a paired t-test
(df=866, t=3.0, p=0.003).


%cp NAMES/utt4.all_auto_withvac.names cross_val.names
%pair CROSS_VAL/PREDICT/utt4.indept.names CROSS_VAL/PREDICT/utt4.all_auto_withvac
% cp ../FEATURES/NAMES/utt1.all_auto_hl_rs.names cross_val.names

Although, the {\sc task-indept} feature sets are a subset of those features used in row 3, it is possible for them to perform better because the {\sc task-indept} features are more
general, and because {\sc ripper} uses a greedy algorithm to discover
its rule sets. For Exchanges 1\&2, the increase from row 3 to row 5 (both of which use {\it auto-SLU-success}) is not significant. Comparing rows 2 and 4, neither of which use {\it auto-SLU-success}, one sees a slight degradation in results for the whole dialogue using {\sc task-indept} features. However, the increase from rows 2 to 5 from 78.1\% to 80.3\% for Exchanges 1\&2 is statistically significant (df=866, t=2.0, p=0.042). This shows that using {\it auto-SLU-success} in combination with the set of {\sc task-indept} features produces a statistically significant increase in accuracy over a set of automatic features that does not include this feature.


The main purpose of these experiments is to determine whether a
dialogue is {\it potentially} problematic, therefore using the whole
dialogue is not useful in a dynamic system. Using Exchanges 1\&2
produces accurate results and would enable the system to adapt in
order to complete the dialogue in the appropriate manner.


%cp ../FEATURES/NAMES/utt4.indept.names cross_val.names




%cp NAMES/utt4.all_auto_withvac.names cross_val.names
%pair CROSS_VAL/PREDICT/utt4.indept.names CROSS_VAL/PREDICT/utt4.all_auto_withvac
% cp ../FEATURES/NAMES/utt1.all_auto_hl_rs.names cross_val.names
%cp NAMES/utt1.all_auto_hlsuccess.names cross_val.name
%cp NAMES/utt1.all_withvac.names cross_val.names
%pair PREDICT/utt1.all_withvac  PREDICT/utt1.all_auto_hlsuccess 

%pair PREDICT/utt2.indept PREDICT/utt2.indept_norsuccess 
%PREDICT/utt2.indept vs PREDICT/utt2.indept_norsuccess W-L-T record:
%        48 wins 32 losses 787 ties
%        prob(win|disagree) 0.6000  +/- 0.0551   1.8143 su>0.5
%pair PREDICT/utt4.indept PREDICT/utt4.indept_norsuccess 
%PREDICT/utt4.indept vs PREDICT/utt4.indept_norsuccess W-L-T record:
%        25 wins 8 losses 834 ties
%        prob(win|disagree) 0.7576  +/- 0.0758   3.4000 su>0.5
%pair PREDICT/utt2.indept PREDICT/utt2.all_auto_withvac
%PREDICT/utt2.indept vs PREDICT/utt2.all_auto_withvac W-L-T record:
%        41 wins 32 losses 794 ties
%        prob(win|disagree) 0.5616  +/- 0.0585   1.0542 su>0.5
%pair PREDICT/utt2.indept PREDICT/utt2.all_auto_withvac 
%PREDICT/utt2.indept vs PREDICT/utt2.all_auto_withvac W-L-T record:
%        41 wins 32 losses 794 ties
%        prob(win|disagree) 0.5616  +/- 0.0585   1.0542 su>0.5
%nero$ pair PREDICT/utt2.indept PREDICT/utt2.all_auto_norsuccess 
%PREDICT/utt2.indept vs PREDICT/utt2.all_auto_norsuccess W-L-T record:
%        53 wins 34 losses 780 ties
%        prob(win|disagree) 0.6092  +/- 0.0526   2.0754 su>0.5

\begin{table}
\centering
\begin{tabular}{|c|l|l|l|l|} \hline
\bf Row & \bf Features &\bf Exchange 1 &\bf Exchange 1\&2& \bf Whole\\ \hline
1&Baseline&67.1&67.1&67.1\\ \hline \hline
2& {\sc auto} (no {\it auto-SLU-success}) &70.1& 78.1 &87.0\\ \hline %all_auto_nr.results 
3& {\sc auto} + {\it auto-SLU-success} &69.6 &79.2 &84.9\\ \hline \hline%all_auto.results 
4& {\sc auto, task-indept} (no {\it auto-SLU-success}) &70.1 & 78.4& 83.4\\ \hline %all_novac.results 
5& {\sc auto, task-indept} + {\it auto-SLU-success} &69.2 & 80.3& 85.4\\ \hline \hline%all_novac.results 
6& {\sc auto} +  {\it SLU-success}& 75.6 &85.7&92.9\\ \hline 
7& {\sc all} ({\sc auto} + Hand-labelled) &77.1 & 86.9 & 91.7\\ \hline%all_novac.results 
\end{tabular}
\caption{Accuracy \% results for predicting problematic dialogues. }
\label{pdp-results-table}
\end{table}

\subsubsection{Hand-labelled Features}

Row 7 in table \ref{pdp-results-table} gives the results using
hand-labelled and automatic features including both {\it SLU-success}
and {\it auto-SLU-success}. By comparing rows 6 and 7, one can see
that there is not very much to be gained by adding the other
hand-labelled features given in Figure \ref{features} to the
hand-labelled and {\it SLU-success} feature set. Only the increase for
Exchange 1 from 75.6\% to 77.1\% is significant (df=866, t=2.3,
p=0.024). For the whole utterance there is actually a degradation of
results from 92.9\% to 91.7\%.

%Table \ref{pdp-results-table} gives results using the feature {\it auto-SLU-success} in both the training and test set. As discussed in Section \ref{exp-design}, an {\it auto-SLU-success} predictor is trained using the cross-validation method in order to derive the automatic feature for the training set. The test set features are derived automatically by the  {\it auto-SLU-success} predictor trained on the whole training set. The hand-labelled  {\it SLU-success} features mentioned in Table \ref{pdp-results-table} are used for both training and testing. 


\begin{table*}[t]
\begin{center}
\begin{tabular}{|r|cccc|} \hline
{\bf Class} & {\bf Occurred } & {\bf Predicted } &{\bf Recall} & {\bf Precision}  \\ \hline 
{\sc tasksuccess}     &  67.0 \%        &  81.7 \%         & 88.1 \%     &  72.5 \%        \\ \hline 
{\sc problematic} &  33.0 \%        &  18.3 \%         & 31.6 \%     & 56.6 \%        \\ \hline 
\end{tabular}
\end{center}
\vspace{-.2mm}
\caption{Precision and Recall with Exchange 1 Automatic Features}
\label{one-ex-table}
\end{table*}

\subsection{Precision and Recall}

The performance of the system that uses automatic features (including
{\it auto-SLU-success}) for the first utterance is given in Table
\ref{one-ex-table}. This system has an overall accuracy of 69.6\%.
These results show that, given the first exchange, the ruleset
predicts that 18.3\% of the dialogues will be problematic, while 33\%
of them actually will be.  Of the problematic dialogues, it can
predict 31.6\% of them.  Once it predicts that a dialogue will be
problematic, it is correct 56.6\% of the time.



\begin{figure*}[t]

\rule{6in}{.2mm} \\
{\bf if} (e2-salience-coverage $\leq$ 0.7)  $\wedge$ (e2-asr-duration $\geq$ 0.04)  $\wedge$ (e2-auto-SLU-success = {\sc no-recog}) {\bf then} {\it problematic} \\ 
{\bf if}  (e2-auto-SLU-success = {\sc rmismatch})$\wedge$  (e2-sys-label = DIAL-FOR-ME) $\wedge$ (e2-asr-duration $\geq$ 3.12) {\bf then} {\it problematic} \\ 
{\bf if} (e1-salience-coverage $\leq$ 0.727) $\wedge$ (e2-salience-coverage $\leq$ 0.706) $\wedge$ (e1-recog  contains  ``help'') $\wedge$ (e1-asr-duration $\leq$ 2.44) {\bf then} {\it problematic} \\ 
{\bf if} (e1-top-confidence $\leq$ 0.924) $\wedge$ (e2-auto-SLU-success = {\sc rmismatch}) $\wedge$ (e2-sys-label = CALLING-CARD) {\bf then} {\it problematic} \\ 
{\bf if} (e1-top-confidence $\leq$ 0.924) $\wedge$ (e2-diff-confidence $\leq$ 0.918) $\wedge$ (e2-collect $\leq$ 0.838) $\wedge$ (e2-asr-duration $\geq$ 9.36) $\wedge$ (e2-dial-for-me $\geq$ 0.5) {\bf then} {\it problematic} \\ 
\rule{6in}{.2mm} \\
\caption{A subset of rules learned by {\sc ripper} when given the automatic features for determining problematic dialogues}
\label{fig:ruleset2}
\vspace{-.2in}
\end{figure*}
%According to these rules, a dialogue will be problematic if the
%confidence score for the top-ranked task (given by the {\sc slu} module) is
%moderate or low and there was touchtone input in the user utterance.
%The second rule says that if the difference between the top confidence
%score and the second-ranked confidence score is moderate or low, and
%the duration of the user utterance is more than 7 seconds, predict
%{\sc problematic}.

\begin{figure*}[t]
\rule{6in}{.2mm} \\
{\bf if} (e2-top-confidence $\leq$ 0.897) $\wedge$  (e2-asr-duration $\geq$ 0.04) $\wedge$  (e2-auto-SLU-success = {\sc no-recog}) {\bf then} {\it problematic} \\ 
{\bf if} (e2-auto-SLU-success = {\sc rmismatch}) $\wedge$  (e2-recog-numwords $\leq$ 7) $\wedge$  (e2-asr-duration $\geq$ 2.28) {\bf then} {\it problematic} \\
{\bf if} (e2-salience-coverage $\leq$ 0.9) $\wedge$  (e2-asr-duration $\geq$ 0.04) $\wedge$  (e2-inconsistency $\geq$ 0.022) $\wedge$  (e1-asr-duration $\leq$ 3.96) $\wedge$  (e2-inconsistency $\geq$ 0.18) {\bf then} {\it problematic} \\
{\bf if} (e1-salience-coverage $\leq$ 0.667) $\wedge$  (e2-salience-coverage $\leq$ 0.692) $\wedge$  (e1-recog contains ``help'') $\wedge$  (e1-asr-duration $\leq$ 7.36) {\bf then} {\it problematic} \\
{\bf if} (e1-top-confidence $\leq$ 0.924) $\wedge$  (e2-diff-confidence $\leq$ 0.918) $\wedge$  (e2-recog contains ``my'') $\wedge$  (e1-asr-duration $\geq$ 4) {\bf then} {\it problematic} \\
{\bf if} (e1-salience-coverage $\leq$ 0.647) $\wedge$  (e1-asr-duration $\geq$ 10.24) $\wedge$  (e2-asr-duration $\geq$ 5.32) {\bf then} {\it problematic} \\
\rule{6in}{.2mm} \\
\caption{A subset of rules learned by {\sc ripper} when given the {\sc task-indept} features for determining problematic dialogues}
\label{fig:ruleset-indept}
\vspace{-.2in}
\end{figure*}





\begin{table*}[t]
\begin{center}
\begin{tabular}{|r|cccc|} \hline
{\bf Class} & {\bf Occurred } & {\bf Predicted } &{\bf Recall} & {\bf Precision}  \\ \hline 
{\sc tasksuccess}     &  67.0 \%        &  80.0 \%         & 94.8 \%     &  79.1 \%        \\ \hline 
{\sc problematic} &  33.0 \%        &  20.0 \%         & 49.5 \%     & 79.7 \%        \\ \hline 
\end{tabular}
\end{center}
\vspace{-.2mm}
\caption{Precision and Recall with Exchange 1\&2 Automatic Features}
%% THESE ARE THE NEW NUMBERS 
\label{two-ex-table}
\end{table*}


The performance of the system that uses automatic features for Exchanges 1\&2 is summarized in Table \ref{two-ex-table}.  These results show that, given the first two exchanges, this ruleset predicts that 20\% of the dialogues will be
problematic, while 33\% of them actually will be.  Of the problematic
dialogues, it can predict 49.5\% of them.  Once it predicts that a
dialogue will be problematic, it is correct 79.7\% of the time.
This classifier has an improvement of 17.87\% in recall and 23.09\% in precision, for an overall improvement in accuracy of 9.6\% over using the first exchange alone.

\subsection{Examination of the Rulesets}
\label{results:rules}
%[HERE]
A subset of the rules from the system that uses automatic features for
Exchanges 1\&2 are given in Figure \ref{fig:ruleset2} (row 3, table
\ref{pdp-results-table}). One observation from these hypotheses is the
classifier's preference for the {\it asr-duration} feature over the
feature for the number of words recognized ({\it recog-numwords}).
One would expect longer utterances to be more difficult, but the
learned rulesets indicate that duration is a better measure of
utterance length than the number of words. Another observation is the
usefulness of the {\sc slu} confidence scores and the {\sc slu}
salience-coverage in predicting problematic dialogues.  These features
seem to provide good general indicators of the system's success in
recognition and understanding.  The fact that the main focus of the
rules is detecting {\sc asr} and {\sc slu} errors and that none of the
{\sc dm} behaviors are used as predictors also indicates that, in all
likelihood, the {\sc dm} is performing as well as it can, given the noisy
input that it is getting from {\sc asr} and {\sc slu}. An alternative
view is that two utterances are not enough to provide meaningful
dialogue features such as counts and percentages of reprompts,
confirmations, etc..


One can see that the top two rules use {\it auto-SLU-success}. The
first rule basically states that if there is no recognition for the
second exchange (as predicted by the {\it auto-SLU-success}) then the
dialogue will fail. The second rule is more interesting as it states
if a misunderstanding has been predicted for the second exchange and
the system label is {\sc DIAL-FOR-ME} and the utterance is long then
the system will fail. In other words, the system frequently
misinterprets long utterances as {\sc DIAL-FOR-ME} resulting in task
failure.

Figure \ref{fig:ruleset-indept} gives a subset of the ruleset for the
{\sc task-indept} feature set for Exchanges 1\&2. One can see a
similarity between this ruleset and the one given in Figure
\ref{fig:ruleset2}. This is due to the fact that when all the
automatic features are available, {\sc ripper} has a tendency to pick
out the more general task-independent ones, with the exception of {\it
sys-label}. If one compares the second rule in both figures, one can
see that {\sc ripper} uses {\it recog-numwords} as a substitute for
the task-specific feature {\it sys-label}.

\subsection{Cross-validation Method vs. Hand-labelled-training Method}
\label{results:hlt}

As mentioned above, 
an alternative to training the PDP on the automatically derived {\it auto-SLU-success} feature is to train it on the hand-labelled {\it SLU-success} while still testing it on the automatic feature. This second method is referred to as ``hand-labelled-training'' and the resulting feature is {\it hlt-SLU-success}. This may provide a more accurate model but it may not capture the characteristics of the automatic feature in the test set. Table \ref{orig_results} gives results for the two methods. One can see from this table that there is a slight, insignificant increase in accuracy for Exchange 1 and the whole dialogue using the hand-labelled-training method. However, the totally automated method yields a better result (79.2\% compared to 77.4\%) for Exchanges 1\&2, which as mentioned above, is the most important result for these experiments. This increase shows a trend but is not significant (df=866, t=1.8, p=0.066). The final row
of the table gives the results using the hand-labelled feature {\it SLU-success} in both the training and testing and is taken as the topline result.

\begin{table}
\centering
\begin{tabular}{|l|l|l|l|} \hline
\bf Features &\bf Exchange 1  &\bf Exchange 1\&2 &\bf Whole \\ \hline
Baseline&67.1&67.1&67.1\\ \hline \hline
%Auto-success&69.0 & 74.1&77.2\\ \hline \hline %only_auto_novac
{\sc auto}&70.1& 78.1 &87.0\\ \hline % all_auto.results 
{\sc auto} + {\it hlt-SLU-success} &70.4 &77.4 &86.2\\ \hline %.auto_novac.result
{\sc auto} + {\it auto-SLU-success} &69.6 &79.2 &84.9\\ \hline %all_auto.results
{\sc auto} + {\it SLU-success}& 75.6 &85.7&92.9\\ \hline %hl_novac.results 
%All features &77.1 & 86.7 & 91.7\\ \hline%all_novac.results 
\end{tabular}
\caption{Accuracy \% results including {\it hlt-SLU-success} derived using the hand-labelled-training method}
\label{orig_results}
\end{table}
%../../FEATURES/ORIG/RESULTS/utt2.just_auto_withvac.results 
%hlt results ~hhastie/HMIHY/data/new_experiments/FEATURES/ORIG/RESULTS/utt2.just_auto_withvac.results
%../../CROSS_VAL/PREDICT/utt2.all_auto_SLU_success
%../../CROSS_VAL/PREDICT/utt2.all_hlt_SLU_success

\subsection{Feature Sets}

It is interesting to examine what types of features are the most discriminatory in determining whether a dialogue is problematic or not. {\sc ripper} was trained separately on sets of features based on the groups given in Figure \ref{features}, namely Acoustic/{\sc asr}, {\sc slu}, Dialogue and Hand-labelled (including {\it SLU-success}). These results are given in Table \ref{feature_results}.

For Exchange 1, only the {\sc slu} features, out of the automatic
feature sets, yields an improvement over the baseline. Interestingly,
training the system on the {\sc asr} yields the best result out of the
automatic feature sets for Exchange 1\&2 and the whole dialogue. These
systems, for example, use {\it asr-duration}, number of recognized
words, and type of recognition grammar as features in their ruleset.

Finally, we give results for the system trained only on {\it
auto-SLU-success} and {\it hlt-SLU-success}. One can see that there is
not much difference in the two sets of results. For Exchanges 1\&2,
the system trained on {\it hlt-SLU-success} has an accuracy which is
significantly higher than the system trained on {\it auto-SLU-success}
by a paired t-test (df=866, t=3.0, p=0.03). On examining the ruleset,
one finds that the {\it hlt-SLU-success} uses {\sc rpartial-mismatch}
where the {\it auto-SLU-success} ruleset does not. The lower accuracy
may be due to the fact that the {\it auto-SLU-success} predictor has a
low recall and precision for {\sc rpartial-mismatch} as seen in Table
\ref{prerec-table}.

%just auto-SLU-success is significantly above the baseline.




\begin{table}
\centering
\begin{tabular}{|l|l|l|l|} \hline
\bf Features &\bf Exchange 1 &\bf Exchange 1\&2&\bf Whole\\ \hline
Baseline&67.1&67.1&67.1\\ \hline \hline
{\sc asr}& 66.7&75.9 &85.6\\ \hline
{\sc slu}& 67.7&71.9 &79.8\\ \hline
Dialogue&65.5 & 74.5 &82.6\\ \hline
Hand-labelled& 76.9 &84.7 &86.2\\ \hline \hline
{\it Auto-SLU-success}&69.0 & 70.9&77.1\\ \hline %auto_rsuccess.results
{\it Hlt-SLU-success}&69.0 & 74.1&77.2\\ \hline  %only_auto_novac
\end{tabular}
\caption{Accuracy \% results for subsets of features}
\label{feature_results}
\end{table}


\subsection{Types of Problematic Dialogues}

As mentioned in Section \ref{exp-sys-data-sec}, there are 3 types of
problematic dialogues: {\sc taskfailure, wizard} and {\sc hangup}.  In
order to determine whether some of these types of problematic
dialogues are more difficult to predict than others, we conducted a
post-hoc analysis of the proportion of prediction failures for each
type of problematic dialogue. Since we were primarily interested in
the performance of the PDP using the full automatic feature set, after
having seen Exchanges 1\&2, we conducted our analysis on this version
of the PDP.  Table \ref{perc_matrix} shows the distribution of the 4
types of dialogue in the test set and whether the Exchanges 1\&2 PDP was
able to predict correctly that the dialogue would be {\sc tasksuccess}
or {\sc problematic}. One can see that the worst performing category is
{\sc taskfailure} and that the PDP predicts incorrectly that  68.5\%
of the {\sc taskfailure} dialogues are {\sc tasksuccess}. 

One reason that this might occur is that this sub-category of
dialogues are much more difficult to predict since in this case the
{\sc hmihy} system has no indication that it is not succeeding in the
task. However, another possibility is that the PDP performs poorly on
this category because there are fewer examples in the training set,
although it does better on the {\sc hangup} subset, which is about the
same proportion. We can eliminate the first possibility by examining
how a learner performs when trained on equal proportions of {\sc
tasksuccess} and {\sc taskfailure} dialogues.  We conducted an
experiment using a subset of {\sc tasksuccess} dialogues in the same
proportion as {\sc taskfailure} for the training and the test set and
trained a second PDP using the fully automatic Exchange 1\&2
features. This resulted in a training set of 690 dialogues and a test
set of 216. The binary classifier has an accuracy of 70\%, the
corresponding recognition matrix is presented in table
\ref{perc_matrix2}. The results show that fewer {\sc taskfailures} are
predicted as successful, suggesting that {\sc taskfailures} are not
inherently more difficult to predict than other classes of problematic
dialogues. Below we discuss the potential of using {\sc ripper}'s loss
ratio to weight different types of classification errors in future
work.

% \begin{table}
% \centering
% \begin{tabular}{|l|l|l|l|} \hline
% \bf True Values&\bf Predicted successful&\bf Predicted problematic&\bf Total\\ \hline
% {\sc tasksuccess}&548&34&582\\ \hline
% {\sc taskfailure}&74&34&108\\ \hline
% {\sc wizard}&43&61&104\\ \hline
% {\sc hangup}&26&47&73\\ \hline
% Total&691&176&867\\ \hline
% \end{tabular}
% \caption{Matrix of recognized {\sc tasksuccess} and {\sc taskfailures}}
% \label{matrix}
% \end{table}

\begin{table}
\centering
\begin{tabular}{|l|l|l|l|} \hline
\bf True Values&\bf Predicted successful&\bf Predicted problematic&\bf total\\ \hline
{\sc tasksuccess}&94.1\% (548) &5.9\% (34) &67.1\% (582) \\ \hline
{\sc taskfailure}&68.5\% (74) &31.5\% (34) &12.0\% (108)\\ \hline
{\sc wizard}&41.3\% (43) &58.7\% (61) &12.5\% (104) \\ \hline
{\sc hangup}&35.6\% (26) &64.4\% (47) &8.4\% (73) \\ \hline
Total&79.7\% (691) &20.3\% (176) &100\% (867) \\ \hline
\end{tabular}
\caption{Matrix of recognized {\sc tasksuccess} and {\sc taskfailures}}
\label{perc_matrix}
\end{table}

\begin{table}
\centering
\begin{tabular}{|l|l|l|l|} \hline
\bf True Values&\bf Predicted successful&\bf Predicted problematic&\bf Total\\ \hline
{\sc tasksuccess}&61.1\% (66) &38.9\% (42) &50\% (108) \\ \hline
{\sc taskfailure}&21.3\% (23) &78.6\% (85) &50\% (108)\\ \hline
Total &41.2\% (89) &58.8\% (127) &100\% (216) \\ \hline
\end{tabular}
\caption{Matrix of recognized {\sc tasksuccess} and {\sc taskfailures} using equal training and testing}
\label{perc_matrix2}
\end{table}


\section{Related Work}
\label{rel-work-sec}

The research reported here is the first that we know of to
automatically analyze a corpus of logs from a spoken dialogue system
for the purpose of learning to {\it predict} problematic situations.
This work builds on two strands of earlier research.  First, this
approach was inspired by work on the {\sc paradise} evaluation
framework for spoken dialogue systems which utilizes both multivariate
linear regression and CART to predict user satisfaction as a function
of a number of other metrics \cite{Walkeretal97,WKL00-jair}. Research using {\sc
paradise} has found that task completion is always a major predictor
of user satisfaction, and has examined predictors of task
completion. Here, our goals are similar in that we attempt to
understand the factors that predict task completion. 
Secondly, this work builds on earlier research on learning to {\it
identify} dialogues in which the user experienced poor speech
recognizer performance \cite{LWK99}. Because that work was based on
features synthesized over the entire dialogue, the hypotheses that
were learned could not be used for prediction during runtime.  In
addition, in contrast to the current study, the previous work
automatically approximated the notion of a good or bad dialogue using
a threshold on the percentage of recognition errors. There is a danger
of this approach being circular when recognition performance at the
utterance level is a primary predictor of a good or bad dialogue.  In
this work, the notion of a good ({\sc tasksuccess}) and bad ({\sc
problematic}) dialogue was labelled by humans.




In previous work, \cite{WLWGL00} reported results from training a
problematic dialogue predictor in which they noted the extent to which
the hand-labelled {\it SLU-success} feature improves classifier
performance. As a result of this prior analysis, in this work we
report results from training an {\it auto-SLU-success} classifier
for each exchange and using its predictions as an input feature to the
Problematic Dialogue Predictor.  There are a number of previous
studies on predicting recognition errors and user corrections which
are related to the {\it auto-SLU-success} predictor that we report on
here
\cite{Hirschbergetal99,HirschICSLP00,HirschNAACL01,Levow98,LitmanNAACL00,SwertsICSLP00}.

\cite{Hirschbergetal99} apply {\sc ripper} to predict recognition
errors in a corpus of 2067 utterances. In contrast to our work,
they utilize prosodic features
in combination with acoustic confidence scores. They report a
best-classifier accuracy of 89\%, which is a 14\% improvement over
their baseline of 74\%.  This result can be compared with our binary
{\it auto-SLU-success} predictor ({\sc rcorrect} vs. {\sc rincorrect})
discussed in Section \ref{results1-sec}.  Examination of the rules
learned by their classifier suggests that durational features are
important. While we do not use amplitude or F0 features, we do have an
{\it asr-duration} feature which is logged by the recognizer.  Without
any of the other prosodic features, the {\it auto-SLU-success}
predictor has an accuracy of 92.4\%, a 29.4\% improvement over the
baseline of 63\%. It is possible that including prosodic features in
the {\it auto-SLU-success} predictor could improve this result even
further.

Previous studies on error correction recognition are also related to
our method of misunderstanding recognition.  \cite{Levow98} applied
similar techniques to learn to distinguish between utterances in which
the user originally provided some information to the system, and {\it
corrections}, which provided the same information a second time,
following a misunderstanding.  This may be more related to our
research than it first appears since corrections are often
misunderstood due to hyper-articulation.  Levow's experiments train a
decision tree using features such as duration, tempo, pitch,
amplitude, and within-utterance pauses.  Examination of the trained
tree in this study also reveals that the durational features are the
most discriminatory. Similarly in our experiments, {\sc ripper} uses
{\it asr-duration} frequently in the developed rule set. Levow obtains an
accuracy rate of 75\% with a baseline of 50\%. 

\cite{SwertsICSLP00} and \cite{HirschNAACL01} perform similar studies
for automatically identifying corrections using prosody, {\sc asr} features
and dialogue context. Corrections are likely to be misrecognized, due
to hyperarticulation. They observe that corrections that are more
distant from the error they correct, are more likely  to
exhibit prosodic differences. Their system automatically
differentiates corrections from non-corrections with an error rate of
15.72\%. Dialogue context is used in the study by \cite{HirschISCA01},
whereby they incorporate whether the user is aware of a mistake at the
current utterance to help predict misunderstandings and misrecognition
of the previous utterances. This study is similar to ours in that they
use a predicted feature about an utterance (the 'aware' feature) to
predict concept or word accuracy, as we use a predicted feature {\it
auto-SLU-success} in the PDP. However, our {\it auto-SLU-success}
feature is automatically available at the time the prediction is being
made, whereas they are making the predictions retroactively. In
addition, they train their system on the hand-labelled feature rather
than the predicted one which they leave as further work.




\cite{Kirchhoff} performs error correction identification using task
independent acoustic and discourse variables. This is a two way
distinction between positive and negative error correction. She uses two
cascaded classifiers, the first is a decision tree trained using 80\%
of the data and validating on 10\%. Examples that have confidence
scores below a threshold go into an exception training set for a
second classifier. During testing, if confidence scores are below a
threshold then the utterance is passed onto the second classifier. She
finds that the most discriminatory features are dialogue context (the
type of previous system utterance) followed by lexical features, with
prosodic features being the least discriminatory. The system
recognizes error corrections with an accuracy of 90\% compared to a
baseline of 81.9\%. In this study \cite{Kirchhoff} deliberately eschews
the use of system specific features, while in our work, we examine the
separate contribution of different feature sets. Our results suggest that
the use of more general features does not negatively impact
performance.

\cite{KrahmerEurospeech99} and \cite{Krahmer99} look at different
features related to responses to problematic system turns. The {\it
disconfirmations} they discuss are responses to explicit or implicit
system verification questions. They observe that disconfirmations are
longer, have a marked word order, and contain specific lexicon such as
``no''. In addition, there are specific prosodic cues such as boundary
tones and pauses. Some of these features such as length, choice of
words are captured in our {\sc ripper} ruleset as discussed above.

As described in Section \ref{slu-classifier}, two methodologies were
compared for incorporating the feature {\it SLU-success} into the
PDP. The first was to use the hand-labelled feature in the training
set, the second to perform separate experiments to predict the feature
for the training set. As the features in the training set are
automatically predicted, it is hoped that the system would pick up the
idiosyncrasies of the noisy data. This training method has been used
previously in \cite{wright_phd} where automatically identified
intonation event features are used to train an automatic speech-act
detector. These automatically derived features provide a better
training model than the hand-labelled ones. This is true also in the
current study as discussed in Section \ref{results2-sec}.


\section{Discussion and Future Work}
\label{fut-sec}

This paper reports results on automatically training a Problematic
Dialogue Predictor to predict problematic human-computer dialogues
using a corpus of 4692 dialogues collected with the {\it How May I
Help You} spoken dialogue system.  The Problematic Dialogue Predictor
can be immediately applied to the system's decision of whether to
transfer the call to a human customer care agent, or be used as a cue
to the system's Dialogue Manager to modify its behavior to repair the
problems identified.  The results show that: (1) Most feature sets
significantly improve over the baseline; (2) Using automatic features
from the whole dialogue, we can identify problematic dialogues 20\%
better than the baseline; (3) Just the first exchange provides
significantly better prediction (3\%) than the baseline; (4) The
second exchange provides an additional significant (13\%) improvement,
(5) A classifier based on task-independent automatic features performs
slightly better than one trained on the full automatic feature set.

The improved ability to predict problematic dialogues is important for
fielding the {\sc hmihy} system without the need for the oversight of
a human customer care agent. These results are promising and we expect
to be able to improve upon them, possibly by incorporating prosody
into the feature set \cite{Hirschbergetal99} or expanding on the {\sc
slu} feature sets.  In addition, the results suggest that the current
PDP is likely to generalize to other dialogue systems.

In future work, we plan to integrate the learned rulesets into the
{\sc hmihy} dialogue system and evaluate the impact that this would
have on the system's overall performance.  There are several ways we
might be able to show this.  Remember that one use of the PDP is to
improve the system's decision of whether and when to transfer a call
to the human customer care agent. The other use would be as input to
the Dialogue Manager's dialogue strategy selection mechanism.
Demonstrating the utility of the PDP for dialogue strategy selection
requires experiments that test out several different ways that this
information could be used by the Dialogue Manager. Demonstrating the
utility of the PDP on the decision to transfer a call necessarily
involves examining the tradeoffs among different kinds of errors.
This is because every call that the {\sc hmihy} system can handle
successfully saves a company the cost of using a human customer care
agent to handle the call. Thus, we can associate this cost with the
decision that {\sc hmihy} makes to transfer the call. When {\sc hmihy}
transfers the call unnecessarily, we call this cost the {\it lost
automation cost}. On the other hand, every call that {\sc hmihy}
attempts to handle and fails, would potentially accrue a different
cost, namely the lost revenue from customers who become irritated with
faulty customer service and take their business elsewhere.  We call
this cost the {\it system failure cost}.  In the results that we
presented here, we report only overall accuracy results and treat {\it
lost automation cost} and {\it system failure cost} as equally
costly. However, in any particular installation of the {\sc hmihy}
system, there may be differences between these costs that would need
to be accounted for in the training of the PDP. It would be possible
to use {\sc ripper} to do this, if these costs were known, by using
its ability to vary the loss ratio.


Another potential issue for future work is the utility of a dialogue
level predictor, e.g. the PDP, vs. an utterance level predictor,
e.g. the {\it auto-SLU-success} predictor, for the goal of automatically
adapting a system's dialogue strategy. This is shown to be effective in \cite{LP00}, where they use a problematic dialogue detector in order to adapt the dialogue strategy for a train enquiry system. It would be possible, and
others have argued \cite{Levow98,Hirschbergetal99,Kirchhoff} that the
dialogue manager's adaptation decisions can be made on the basis of
local behavior, i.e. on the basis of recognizing that the current
utterance has been misunderstood, or that the current utterance is a
correction. However, it is clear that the decision to transfer the
call to a human customer care agent cannot be made on the basis of
only local information because the system can often recover from a
single error. Thus, we expect that the ability to be able to predict the
dialogue outcome as we do here will continue to be important even in
systems that use local predictors for understanding and correction.



\section{Acknowledgments}
Thanks to Ron Prass, Diane Litman, Richard Sutton, Mazin Rahim and
Michael Kearns for discussions on various aspects of this work.


\bibliographystyle{theapa}

%\bibliography{lyn}
%\bibliography{/usr/walker/nl}
\begin{thebibliography}{}

\bibitem[\protect\BCAY{Abella \BBA\ Gorin}{Abella \BBA\
  Gorin}{1999}]{AbellaGorin}
Abella, A.\BBACOMMA\  \BBA\ Gorin, A. \BBOP1999\BBCP.
\newblock \BBOQ Construct algebra: An analytical method for dialog
  management\BBCQ\
\newblock In {\Bem Proceedings of Thirty Seventh Annual Meeting of the
  Association for Computational Linguistics}.

\bibitem[\protect\BCAY{Baggia, Castagneri, \BBA\ Danieli}{Baggia
  et~al.}{1998}]{Baggia98}
Baggia, P., Castagneri, G., \BBA\ Danieli, M. \BBOP1998\BBCP.
\newblock \BBOQ {Field Trials of the Italian ARISE Train Timetable
  System}\BBCQ\
\newblock In {\Bem {Interactive Voice Technology for Telecommunications
  Applications, IVTTA}}, \BPGS\ 97--102.

\bibitem[\protect\BCAY{Brieman, Friedman, Olshen, \BBA\ Stone}{Brieman
  et~al.}{1984}]{Brieman}
Brieman, L., Friedman, J.~H., Olshen, R.~A., \BBA\ Stone, C.~J. \BBOP1984\BBCP.
\newblock {\Bem Classification and Regression Trees}.
\newblock Wadsworth and Brooks, Monterey California.

\bibitem[\protect\BCAY{Catlett}{Catlett}{1991}]{Catlett91}
Catlett, J. \BBOP1991\BBCP.
\newblock \BBOQ Megainduction: A test flight\BBCQ\
\newblock In {\Bem Proceedings of the Eighth International Conference on
  Machine Learning}.

\bibitem[\protect\BCAY{Chu-Carroll \BBA\ Carpenter}{Chu-Carroll \BBA\
  Carpenter}{1999}]{ChuCarpenter99}
Chu-Carroll, J.\BBACOMMA\  \BBA\ Carpenter, B. \BBOP1999\BBCP.
\newblock \BBOQ Vector-based natural language call routing\BBCQ\
\newblock {\Bem Computational Linguistics}, {\Bem 25-3}, 361--387.

\bibitem[\protect\BCAY{Cohen}{Cohen}{1995}]{CohenML95}
Cohen, W. \BBOP1995\BBCP.
\newblock \BBOQ Fast effective rule induction\BBCQ\
\newblock In {\Bem Proceedings of the Twelfth International Conference on
  Machine Learning}.

\bibitem[\protect\BCAY{Cohen}{Cohen}{1996}]{CohenAAAI96}
Cohen, W. \BBOP1996\BBCP.
\newblock \BBOQ Learning trees and rules with set-valued features\BBCQ\
\newblock In {\Bem Fourteenth Conference of the American Association of
  Artificial Intelligence}.

\bibitem[\protect\BCAY{E.~Ammicht \BBA\ Alonso}{E.~Ammicht \BBA\
  Alonso}{1999}]{AGA99}
E.~Ammicht, A.~G.\BBACOMMA\  \BBA\ Alonso, T. \BBOP1999\BBCP.
\newblock \BBOQ Knowledge collection for natural language spoken dialog
  systems\BBCQ\
\newblock In {\Bem Proceedings of the European Conference on Speech
  Communication and Technology}.

\bibitem[\protect\BCAY{Furnkranz \BBA\ Widmer}{Furnkranz \BBA\
  Widmer}{1994}]{FurnkranzWidmer94}
Furnkranz, J.\BBACOMMA\  \BBA\ Widmer, G. \BBOP1994\BBCP.
\newblock \BBOQ Incremental reduced error pruning\BBCQ\
\newblock In {\Bem Proceedings of the Eleventh National Conference on Machine
  Learning}.

\bibitem[\protect\BCAY{Gorin, Riccardi, \BBA\ Wright}{Gorin
  et~al.}{1997}]{GRW97}
Gorin, A., Riccardi, G., \BBA\ Wright, J. \BBOP1997\BBCP.
\newblock \BBOQ {How May I Help You?}\BBCQ\
\newblock {\Bem Speech Communication}, {\Bem 23}, 113--127.

\bibitem[\protect\BCAY{Hirschberg, Litman, \BBA\ Swerts}{Hirschberg
  et~al.}{1999}]{Hirschbergetal99}
Hirschberg, J.~B., Litman, D.~J., \BBA\ Swerts, M. \BBOP1999\BBCP.
\newblock \BBOQ Prosodic cues to recognition errors\BBCQ\
\newblock In {\Bem Proc. of the Automatic Speech Recognition and Understanding
  Workshop}.

\bibitem[\protect\BCAY{Hirschberg, Litman, \BBA\ Swerts}{Hirschberg
  et~al.}{2000}]{HirschICSLP00}
Hirschberg, J.~B., Litman, D.~J., \BBA\ Swerts, M. \BBOP2000\BBCP.
\newblock \BBOQ Generalizing prosodic prediction of speech recognition
  errors\BBCQ\
\newblock In {\Bem Proceedings of the 6th International Conference of Spoken
  Language Processing (ICSLP-2000)}.

\bibitem[\protect\BCAY{Hirschberg, Litman, \BBA\ Swerts}{Hirschberg
  et~al.}{2001a}]{HirschISCA01}
Hirschberg, J.~B., Litman, D.~J., \BBA\ Swerts, M. \BBOP2001a\BBCP.
\newblock \BBOQ Detecting misrecognitions and corrections in spoken dialogue
  systems from 'aware' sites\BBCQ\
\newblock In {\Bem Proceedings of the Workshop on Prosody in Speech Recognition
  and Understanding}.

\bibitem[\protect\BCAY{Hirschberg, Litman, \BBA\ Swerts}{Hirschberg
  et~al.}{2001b}]{HirschNAACL01}
Hirschberg, J.~B., Litman, D.~J., \BBA\ Swerts, M. \BBOP2001b\BBCP.
\newblock \BBOQ Identifying user corrections automatically in spoken dialogue
  system\BBCQ\
\newblock In {\Bem Proceedings of the Second Meeting of the North American
  Chapter of the Association for Computational Linguistics}.

\bibitem[\protect\BCAY{Kirchhoff}{Kirchhoff}{2001}]{Kirchhoff}
Kirchhoff, K. \BBOP2001\BBCP.
\newblock \BBOQ A comparison of classification techniques for the automatic
  detection of error corrections in human-computer dialogues\BBCQ\
\newblock In {\Bem Proceedings of the North American Meeting of the NAACL
  Workshop on Adaptation in DIalogue Systems}.

\bibitem[\protect\BCAY{Krahmer, Swerts, Theune, \BBA\ Weegels}{Krahmer
  et~al.}{1999a}]{KrahmerEurospeech99}
Krahmer, E., Swerts, M., Theune, M., \BBA\ Weegels, M. \BBOP1999a\BBCP.
\newblock \BBOQ Problem spotting in human-machine interaction\BBCQ\
\newblock In {\Bem Proc. Eurospeech 99}.

\bibitem[\protect\BCAY{Krahmer, Swerts, Theune, \BBA\ Weegels}{Krahmer
  et~al.}{1999b}]{Krahmer99}
Krahmer, E., Swerts, M., Theune, M., \BBA\ Weegels, M. \BBOP1999b\BBCP.
\newblock \BBOQ Prosodic correlates of disconfirmations\BBCQ\
\newblock In {\Bem ESCA Workshop on Interactive Dialogue in Multi-Modal
  Systems}.

\bibitem[\protect\BCAY{Langkilde, Walker, Wright, Gorin, \BBA\
  Litman}{Langkilde et~al.}{1999}]{LWRGL99}
Langkilde, I., Walker, M.~A., Wright, J., Gorin, A., \BBA\ Litman, D.
  \BBOP1999\BBCP.
\newblock \BBOQ Automatic prediction of problematic human-computer dialogues in
  {How May I Help You}?\BBCQ\
\newblock In {\Bem Proceedings of the IEEE Workshop on Automatic Speech
  Recognition and Understanding, ASRUU99}.

\bibitem[\protect\BCAY{Levow}{Levow}{1998}]{Levow98}
Levow, G.-A. \BBOP1998\BBCP.
\newblock \BBOQ Characterizing and recognizing spoken corrections in
  human-computer dialogue\BBCQ\
\newblock In {\Bem Proceedings of the 36th Annual Meeting of the Association of
  Computational Linguistics}, \BPGS\ 736--742.

\bibitem[\protect\BCAY{Litman, Hirschberg, \BBA\ Swerts}{Litman
  et~al.}{2000}]{LitmanNAACL00}
Litman, D.~J., Hirschberg, J.~B., \BBA\ Swerts, M. \BBOP2000\BBCP.
\newblock \BBOQ Predicting automatic speech recognition performance using
  prosodic cues\BBCQ\
\newblock In {\Bem Proceedings of the First Meeting of the North American
  Chapter of the Association for Computational Linguistics}.

\bibitem[\protect\BCAY{Litman \BBA\ Pan}{Litman \BBA\ Pan}{2000}]{LP00}
Litman, D.~J.\BBACOMMA\  \BBA\ Pan, S. \BBOP2000\BBCP.
\newblock \BBOQ Predicting and adapting to poor speech recognition in a spoken
  dialogue system\BBCQ\
\newblock In {\Bem {Proc. of the Seventeenth National Conference on Artificial
  Intelligence, AAAI-2000}}.

\bibitem[\protect\BCAY{Litman, Walker, \BBA\ Kearns}{Litman
  et~al.}{1999}]{LWK99}
Litman, D.~J., Walker, M.~A., \BBA\ Kearns, M.~J. \BBOP1999\BBCP.
\newblock \BBOQ Automatic detection of poor speech recognition at the dialogue
  level\BBCQ\
\newblock In {\Bem Proceedings of the Thirty Seventh Annual Meeting of the
  Association of Computational Linguistics}, \BPGS\ 309--316.

\bibitem[\protect\BCAY{Riccardi \BBA\ Gorin}{Riccardi \BBA\ Gorin}{2000}]{RG}
Riccardi, G.\BBACOMMA\  \BBA\ Gorin, A. \BBOP2000\BBCP.
\newblock \BBOQ Spoken language adaptation over time and state in a natural
  spoken dialog system\BBCQ\
\newblock {\Bem IEEE Transactions on Speech and Audio Processing}, {\Bem
  8\/}(1), 3--10.

\bibitem[\protect\BCAY{Sanderman, Sturm, den Os, Boves, \BBA\
  Cremers}{Sanderman et~al.}{1998}]{Sanderman98}
Sanderman, A., Sturm, J., den Os, E., Boves, L., \BBA\ Cremers, A.
  \BBOP1998\BBCP.
\newblock \BBOQ Evaluation of the dutchtrain timetable information system
  developed in the {ARISE} project\BBCQ\
\newblock In {\Bem Interactive Voice Technology for Telecommunications
  Applications, IVTTA}, \BPGS\ 91--96.

\bibitem[\protect\BCAY{Seneff, Zue, Polifroni, Pao, Hetherington, Goddeau,
  \BBA\ Glass}{Seneff et~al.}{1995}]{Seneff95}
Seneff, S., Zue, V., Polifroni, J., Pao, C., Hetherington, L., Goddeau, D.,
  \BBA\ Glass, J. \BBOP1995\BBCP.
\newblock \BBOQ The preliminary development of a displayless {PEGASUS}
  system\BBCQ\
\newblock In {\Bem ARPA Spoken Language Technology Workshop}.

\bibitem[\protect\BCAY{Shriberg, Wade, \BBA\ Price}{Shriberg
  et~al.}{1992}]{SWP92}
Shriberg, E., Wade, E., \BBA\ Price, P. \BBOP1992\BBCP.
\newblock \BBOQ Human-machine problem solving using spoken language systems
  ({SLS}): Factors affecting performance and user satisfaction\BBCQ\
\newblock In {\Bem Proceedings of the DARPA Speech and NL Workshop}, \BPGS\
  49--54.

\bibitem[\protect\BCAY{Swerts, Litman, \BBA\ Hirschberg}{Swerts
  et~al.}{2000}]{SwertsICSLP00}
Swerts, M., Litman, D.~J., \BBA\ Hirschberg, J.~B. \BBOP2000\BBCP.
\newblock \BBOQ Corrections in spoken dialogue systems\BBCQ\
\newblock In {\Bem Proceedings of the 6th International Conference of Spoken
  Language Processing (ICSLP-2000)}.

\bibitem[\protect\BCAY{Walker, Fromer, \BBA\ Narayanan}{Walker
  et~al.}{1998}]{WFN98}
Walker, M.~A., Fromer, J.~C., \BBA\ Narayanan, S. \BBOP1998\BBCP.
\newblock \BBOQ Learning optimal dialogue strategies: A case study of a spoken
  dialogue agent for email\BBCQ\
\newblock In {\Bem Proceedings of the 36th Annual Meeting of the Association of
  Computational Linguistics, COLING/ACL 98}, \BPGS\ 1345--1352.


\bibitem[\protect\BCAY{Walker, Kamm, \BBA\ Litman}{Walker
  et~al.}{2000a}]{WKL00-jair}
Walker, M.~A., Kamm, C.~A., \BBA\ Litman, D.~J. \BBOP2000a\BBCP.
\newblock \BBOQ Towards developing general models of usability with
  {PARADISE}\BBCQ\
\newblock In {\Bem Natural Language Engineering: Special Issue on Best Practice
  in Spoken Dialogue Systems}.

\bibitem[\protect\BCAY{Walker, Langkilde, Wright, Gorin, \BBA\ Litman}{Walker
  et~al.}{2000b}]{WLWGL00}
Walker, M.~A., Langkilde, I., Wright, J., Gorin, A., \BBA\ Litman, D.
  \BBOP2000b\BBCP.
\newblock \BBOQ {Learning to Predict Problematic Situations in a Spoken
  Dialogue System: Experiments with How May I Help You?}\BBCQ\
\newblock In {\Bem Proceedings of the North American Meeting of the Association
  for Computational Linguistics}.

\bibitem[\protect\BCAY{Walker, Litman, Kamm, \BBA\ Abella}{Walker
  et~al.}{1997}]{Walkeretal97}
Walker, M.~A., Litman, D., Kamm, C.~A., \BBA\ Abella, A. \BBOP1997\BBCP.
\newblock \BBOQ {PARADISE}: A general framework for evaluating spoken dialogue
  agents\BBCQ\
\newblock In {\Bem Proceedings of the 35th Annual Meeting of the Association of
  Computational Linguistics, ACL/EACL 97}, \BPGS\ 271--280.

%% HAD TO HAND EDIT HERE TO GET IT TO PUT 2000C and put things
%% in the right order!!
\bibitem[\protect\BCAY{Walker, Wright, \BBA\ Langkilde}{Walker
  et~al.}{2000c}]{WLW-icml00}
Walker, M.~A., Wright, J., \BBA\ Langkilde, I. \BBOP2000c\BBCP.
\newblock \BBOQ Using natural language processing and discourse features to
  identify understanding errors in a spoken dialogue system\BBCQ\
\newblock In {\Bem Proceedings of the Seventeenth International Conference on
  Machine Learning}.

\bibitem[\protect\BCAY{Weiss \BBA\ Kulikowski}{Weiss \BBA\
  Kulikowski}{1991}]{KulikowskiBook90}
Weiss, S.~M.\BBACOMMA\  \BBA\ Kulikowski, C. \BBOP1991\BBCP.
\newblock {\Bem Computer Systems That Learn: Classification and Prediction
  Methods from Statistics, Neural Nets, Machine Learning, and Expert Systems}.
\newblock San Mateo, CA: Morgan Kaufmann.

\bibitem[\protect\BCAY{Wright}{Wright}{2000}]{wright_phd}
Wright, H. \BBOP2000\BBCP.
\newblock {\Bem Modelling Prosodic and Dialogue Information for Automatic
  Speech Recognition}.
\newblock Ph.D.\ thesis, University of Edinburgh.

\end{thebibliography}

\end{document}


