\begin{thebibliography}{100}

\bibitem{Ackley90b}
David~H. Ackley and Michael~L. Littman.
\newblock Generalization and scaling in reinforcement learning.
\newblock In D.~S. Touretzky, editor, {\em Advances in Neural Information
  Processing Systems 2}, pages 550--557, San Mateo, CA, 1990. Morgan Kaufmann.

\bibitem{albs:anwa}
J.~S. Albus.
\newblock A new approach to manipulator control: Cerebellar model articulation
  controller (cmac).
\newblock {\em Journal of Dynamic Systems, Measurement and Control},
  97:220--227, 1975.

\bibitem{Albus81}
James~S. Albus.
\newblock {\em Brains, Behavior, and Robotics}.
\newblock BYTE Books, Subsidiary of McGraw-Hill, Peterborough, New Hampshire,
  1981.

\bibitem{Anderson86}
Charles~W. Anderson.
\newblock {\em Learning and Problem Solving with Multilayer Connectionist
  Systems}.
\newblock PhD thesis, University of Massachusetts, Amherst, MA, 1986.

\bibitem{Ashar94}
Rachita~(Ronny) Ashar.
\newblock Hierarchical learning in stochastic domains.
\newblock Master's thesis, Brown University, Providence, Rhode Island, 1994.

\bibitem{Baird95}
Leemon Baird.
\newblock Residual algorithms: {R}einforcement learning with function
  approximation.
\newblock In Armand Prieditis and Stuart Russell, editors, {\em Proceedings of
  the Twelfth International Conference on Machine Learning}, pages 30--37, San
  Francisco, CA, 1995. Morgan Kaufmann.

\bibitem{Baird93}
Leemon~C. Baird and A.~H. Klopf.
\newblock Reinforcement learning with high-dimensional, continuous actions.
\newblock Technical Report WL-TR-93-1147, Wright-Patterson Air Force Base Ohio:
  {W}right Laboratory, 1993.

\bibitem{Barto95}
Andrew~G. Barto, S.~J. Bradtke, and Satinder~P. Singh.
\newblock Learning to act using real-time dynamic programming.
\newblock {\em Artificial Intelligence}, 72(1):81--138, 1995.

\bibitem{Barto83}
Andrew~G. Barto, Richard~S. Sutton, and Charles~W. Anderson.
\newblock Neuronlike adaptive elements that can solve difficult learning
  control problems.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
  SMC-13(5):834--846, 1983.

\bibitem{Bellman57}
Richard Bellman.
\newblock {\em Dynamic Programming}.
\newblock Princeton University Press, Princeton, NJ, 1957.

\bibitem{Berenji91}
Hamid~R. Berenji.
\newblock Artificial neural networks and approximate reasoning for intelligent
  control in space.
\newblock In {\em American Control Conference}, pages 1075--1080, 1991.

\bibitem{Berry85}
Donald~A. Berry and Bert Fristedt.
\newblock {\em Bandit Problems: {S}equential Allocation of Experiments}.
\newblock Chapman and Hall, London, UK, 1985.

\bibitem{Bertsekas87}
Dimitri~P. Bertsekas.
\newblock {\em Dynamic Programming: {D}eterministic and Stochastic Models}.
\newblock Prentice-Hall, Englewood Cliffs, NJ, 1987.

\bibitem{Bertsekas95}
Dimitri~P. Bertsekas.
\newblock {\em Dynamic Programming and Optimal Control}.
\newblock Athena Scientific, Belmont, Massachusetts, 1995.
\newblock Volumes 1 and 2.

\bibitem{Bertsekas89b}
Dimitri~P. Bertsekas and D.~A. Casta\~{n}on.
\newblock Adaptive aggregation for infinite horizon dynamic programming.
\newblock {\em IEEE Transactions on Automatic Control}, 34(6):589--598, 1989.

\bibitem{Bertsekas89}
Dimitri~P. Bertsekas and John~N. Tsitsiklis.
\newblock {\em Parallel and Distributed Computation: {N}umerical Methods}.
\newblock Prentice-Hall, Englewood Cliffs, NJ, 1989.

\bibitem{Box87}
G.~E.~P. Box and N.~R. Draper.
\newblock {\em Empirical Model-Building and Response Surfaces}.
\newblock Wiley, 1987.

\bibitem{Boyan94b}
Justin~A. Boyan and Andrew~W. Moore.
\newblock Generalization in reinforcement learning: {S}afely approximating the
  value function.
\newblock In G.~Tesauro, D.~S. Touretzky, and T.~K. Leen, editors, {\em
  Advances in Neural Information Processing Systems 7}, Cambridge, MA, 1995.
  The MIT Press.

\bibitem{brgh:intr}
D.~Burghes and A.~Graham.
\newblock {\em Introduction to Control Theory including Optimal Control}.
\newblock Ellis Horwood, 1980.

\bibitem{Cassandra94}
Anthony~R. Cassandra, Leslie~Pack Kaelbling, and Michael~L. Littman.
\newblock Acting optimally in partially observable stochastic domains.
\newblock In {\em Proceedings of the Twelfth National Conference on Artificial
  Intelligence}, Seattle, WA, 1994.

\bibitem{Chapman91}
David Chapman and Leslie~Pack Kaelbling.
\newblock Input generalization in delayed reinforcement learning: {A}n
  algorithm and performance comparisons.
\newblock In {\em Proceedings of the International Joint Conference on
  Artificial Intelligence}, Sydney, Australia, 1991.

\bibitem{Chrisman92a}
Lonnie Chrisman.
\newblock Reinforcement learning with perceptual aliasing: {T}he perceptual
  distinctions approach.
\newblock In {\em Proceedings of the Tenth National Conference on Artificial
  Intelligence}, pages 183--188, San Jose, CA, 1992. AAAI Press.

\bibitem{Chrisman93}
Lonnie Chrisman and Michael Littman.
\newblock Hidden state and short-term memory, 1993.
\newblock Presentation at Reinforcement Learning Workshop, Machine Learning
  Conference.

\bibitem{Cichosz95}
Pawel Cichosz and Jan~J. Mulawka.
\newblock Fast and efficient reinforcement learning with truncated temporal
  differences.
\newblock In Armand Prieditis and Stuart Russell, editors, {\em Proceedings of
  the Twelfth International Conference on Machine Learning}, pages 99--107, San
  Francisco, CA, 1995. Morgan Kaufmann.

\bibitem{Cleveland88}
W.~S. Cleveland and S.~J. Delvin.
\newblock Locally weighted regression: {A}n approach to regression analysis by
  local fitting.
\newblock {\em Journal of the American Statistical Association},
  83(403):596--610, September 1988.

\bibitem{Cliff95}
Dave Cliff and Susi Ross.
\newblock Adding temporary memory to {ZCS}.
\newblock {\em Adaptive Behavior}, 3(2):101--150, 1994.

\bibitem{Condon92}
Anne Condon.
\newblock The complexity of stochastic games.
\newblock {\em Information and Computation}, 96(2):203--224, February 1992.

\bibitem{Connell93}
Jonathan Connell and Sridhar Mahadevan.
\newblock Rapid task learning for real robots.
\newblock In {\em Robot Learning}. Kluwer Academic Publishers, 1993.

\bibitem{cr:impr}
R.~H. Crites and A.~G. Barto.
\newblock Improving elevator performance using reinforcement learning.
\newblock In D.~Touretzky, M.~Mozer, and M.~Hasselmo, editors, {\em Neural
  Information Processing Systems 8}, 1996.

\bibitem{Dayan92}
Peter Dayan.
\newblock The convergence of {TD($\lambda$)} for general {$\lambda$}.
\newblock {\em Machine Learning}, 8(3):341--362, 1992.

\bibitem{Dayan93}
Peter Dayan and Geoffrey~E. Hinton.
\newblock Feudal reinforcement learning.
\newblock In S.~J. Hanson, J.~D. Cowan, and C.~L. Giles, editors, {\em Advances
  in Neural Information Processing Systems 5}, San Mateo, CA, 1993. Morgan
  Kaufmann.

\bibitem{Dayan94}
Peter Dayan and Terrence~J. Sejnowski.
\newblock {TD$(\lambda)$} converges with probability 1.
\newblock {\em Machine Learning}, 14(3), 1994.

\bibitem{Dean93}
Thomas Dean, Leslie~Pack Kaelbling, Jak Kirman, and Ann Nicholson.
\newblock Planning with deadlines in stochastic domains.
\newblock In {\em Proceedings of the Eleventh National Conference on Artificial
  Intelligence}, Washington, DC, 1993.

\bibitem{DEpenoux63}
F.~{D'E}penoux.
\newblock A probabilistic production and inventory problem.
\newblock {\em Management Science}, 10:98--108, 1963.

\bibitem{Derman70}
Cyrus Derman.
\newblock {\em Finite State {M}arkovian Decision Processes}.
\newblock Academic Press, New York, 1970.

\bibitem{Dorigo94}
M.~Dorigo and H.~Bersini.
\newblock A comparison of q-learning and classifier systems.
\newblock In {\em From Animals to Animats: {P}roceedings of the Third
  International Conference on the Simulation of Adaptive Behavior}, Brighton,
  UK, 1994.

\bibitem{Dorigo95a}
M.~Dorigo and M.~Colombetti.
\newblock Robot shaping: {D}eveloping autonomous agents through learning.
\newblock {\em Artificial Intelligence}, 71(2):321--370, December 1994.

\bibitem{Dorigo95}
Marco Dorigo.
\newblock Alecsys and the {AutonoMouse}: {L}earning to control a real robot by
  distributed classifier systems.
\newblock {\em Machine Learning}, 19, 1995.

\bibitem{Fiechter94}
Claude-Nicolas Fiechter.
\newblock Efficient reinforcement learning.
\newblock In {\em Proceedings of the Seventh Annual {ACM} Conference on
  Computational Learning Theory}, pages 88--97. Association of Computing
  Machinery, 1994.

\bibitem{Gittins89}
J.~C. Gittins.
\newblock {\em Multi-armed Bandit Allocation Indices}.
\newblock Wiley-Interscience series in systems and optimization. Wiley,
  Chichester, NY, 1989.

\bibitem{Goldberg89}
D.~Goldberg.
\newblock {\em Genetic algorithms in search, optimization, and machine
  learning}.
\newblock Addison-Wesley, MA, 1989.

\bibitem{Gordon95}
Geoffrey~J. Gordon.
\newblock Stable function approximation in dynamic programming.
\newblock In Armand Prieditis and Stuart Russell, editors, {\em Proceedings of
  the Twelfth International Conference on Machine Learning}, pages 261--268,
  San Francisco, CA, 1995. Morgan Kaufmann.

\bibitem{Gullapalli90}
Vijay Gullapalli.
\newblock A stochastic reinforcement learning algorithm for learning
  real-valued functions.
\newblock {\em Neural Networks}, 3:671--692, 1990.

\bibitem{Gullapalli92}
Vijay Gullapalli.
\newblock {\em Reinforcement learning and its application to control}.
\newblock PhD thesis, University of Massachusetts, Amherst, MA, 1992.

\bibitem{Hilgard75}
Ernest~R. Hilgard and Gordon~H. Bower.
\newblock {\em Theories of Learning}.
\newblock Prentice-Hall, Englewood Cliffs, NJ, fourth edition, 1975.

\bibitem{Hoffman66}
A.~J. Hoffman and R.~M. Karp.
\newblock On nonterminating stochastic games.
\newblock {\em Management Science}, 12:359--370, 1966.

\bibitem{Holland75}
John~H. Holland.
\newblock {\em Adaptation in Natural and Artificial Systems}.
\newblock University of Michigan Press, Ann Arbor, MI, 1975.

\bibitem{Howard60}
Ronald~A. Howard.
\newblock {\em Dynamic Programming and {M}arkov Processes}.
\newblock The MIT Press, Cambridge, MA, 1960.

\bibitem{Jaakkola94}
Tommi Jaakkola, Michael~I. Jordan, and Satinder~P. Singh.
\newblock On the convergence of stochastic iterative dynamic programming
  algorithms.
\newblock {\em Neural Computation}, 6(6), November 1994.

\bibitem{Jaakkola95}
Tommi Jaakkola, Satinder~Pal Singh, and Michael~I. Jordan.
\newblock Monte-carlo reinforcement learning in non-{M}arkovian decision
  problems.
\newblock In G.~Tesauro, D.~S. Touretzky, and T.~K. Leen, editors, {\em
  Advances in Neural Information Processing Systems 7}, Cambridge, MA, 1995.
  The MIT Press.

\bibitem{Kaelbling93a}
Leslie~Pack Kaelbling.
\newblock Hierarchical learning in stochastic domains: {P}reliminary results.
\newblock In {\em Proceedings of the Tenth International Conference on Machine
  Learning}, Amherst, MA, 1993. Morgan Kaufmann.

\bibitem{Kaelbling93}
Leslie~Pack Kaelbling.
\newblock {\em Learning in Embedded Systems}.
\newblock The MIT Press, Cambridge, MA, 1993.

\bibitem{Kaelbling94a}
Leslie~Pack Kaelbling.
\newblock Associative reinforcement learning: {A} generate and test algorithm.
\newblock {\em Machine Learning}, 15(3), 1994.

\bibitem{Kaelbling94b}
Leslie~Pack Kaelbling.
\newblock Associative reinforcement learning: {F}unctions in $k$-{DNF}.
\newblock {\em Machine Learning}, 15(3), 1994.

\bibitem{Kirman94}
Jak Kirman.
\newblock {\em Predicting Real-Time Planner Performance by Domain
  Characterization}.
\newblock PhD thesis, Department of Computer Science, Brown University, 1994.

\bibitem{Koenig93}
Sven Koenig and Reid~G. Simmons.
\newblock Complexity analysis of real-time reinforcement learning.
\newblock In {\em Proceedings of the Eleventh National Conference on Artificial
  Intelligence}, pages 99--105, Menlo Park, California, 1993. AAAI Press/MIT
  Press.

\bibitem{Kumar86}
P.~R. Kumar and P.~P. Varaiya.
\newblock {\em Stochastic Systems: Estimation, Identification, and Adaptive
  Control}.
\newblock Prentice Hall, Englewood Cliffs, New Jersey, 1986.

\bibitem{Lee91}
C.~C. Lee.
\newblock A self learning rule-based controller employing approximate reasoning
  and neural net concepts.
\newblock {\em International Journal of Intelligent Systems}, 6(1):71--93,
  1991.

\bibitem{Lin91}
Long-Ji Lin.
\newblock Programming robots using reinforcement learning and teaching.
\newblock In {\em Proceedings of the Ninth National Conference on Artificial
  Intelligence}, 1991.

\bibitem{Lin93}
Long-Ji Lin.
\newblock Hierachical learning of robot skills by reinforcement.
\newblock In {\em Proceedings of the International Conference on Neural
  Networks}, 1993.

\bibitem{Lin93b}
Long-Ji Lin.
\newblock {\em Reinforcement Learning for Robots Using Neural Networks}.
\newblock PhD thesis, Carnegie Mellon University, Pittsburgh, PA, 1993.

\bibitem{LinMitchell92}
Long-Ji Lin and Tom~M. Mitchell.
\newblock Memory approaches to reinforcement learning in non-{M}arkovian
  domains.
\newblock Technical Report CMU-CS-92-138, Carnegie Mellon University, School of
  Computer Science, May 1992.

\bibitem{Littman94b}
Michael~L. Littman.
\newblock {M}arkov games as a framework for multi-agent reinforcement learning.
\newblock In {\em Proceedings of the Eleventh International Conference on
  Machine Learning}, pages 157--163, San Francisco, CA, 1994. Morgan Kaufmann.

\bibitem{Littman94}
Michael~L. Littman.
\newblock Memoryless policies: {T}heoretical limitations and practical results.
\newblock In Dave Cliff, Philip Husbands, Jean-Arcady Meyer, and Stewart~W.
  Wilson, editors, {\em From Animals to Animats 3: {P}roceedings of the Third
  International Conference on Simulation of Adaptive Behavior}, Cambridge, MA,
  1994. The MIT Press.

\bibitem{Littman95f}
Michael~L. Littman, Anthony Cassandra, and Leslie~Pack Kaelbling.
\newblock Learning policies for partially observable environments: {S}caling
  up.
\newblock In Armand Prieditis and Stuart Russell, editors, {\em Proceedings of
  the Twelfth International Conference on Machine Learning}, pages 362--370,
  San Francisco, CA, 1995. Morgan Kaufmann.

\bibitem{Littman95e}
Michael~L. Littman, Thomas~L. Dean, and Leslie~Pack Kaelbling.
\newblock On the complexity of solving {M}arkov decision problems.
\newblock In {\em Proceedings of the Eleventh Annual Conference on Uncertainty
  in Artificial Intelligence (UAI--95)}, Montreal, Qu\'{e}bec, Canada, 1995.

\bibitem{Lovejoy91}
William~S. Lovejoy.
\newblock A survey of algorithmic methods for partially observable {M}arkov
  decision processes.
\newblock {\em Annals of Operations Research}, 28:47--66, 1991.

\bibitem{Maes90}
Pattie Maes and Rodney~A. Brooks.
\newblock Learning to coordinate behaviors.
\newblock In {\em Proceedings Eighth National Conference on Artificial
  Intelligence}, pages 796--802. Morgan Kaufmann, 1990.

\bibitem{Mahadevan94b}
Sridhar Mahadevan.
\newblock To discount or not to discount in reinforcement learning: {A} case
  study comparing {R} learning and {Q} learning.
\newblock In {\em Proceedings of the Eleventh International Conference on
  Machine Learning}, pages 164--172, San Francisco, CA, 1994. Morgan Kaufmann.

\bibitem{Mahadevan96}
Sridhar Mahadevan.
\newblock Average reward reinforcement learning: {F}oundations, algorithms, and
  empirical results.
\newblock {\em Machine Learning}, 22(1), 1996.

\bibitem{Mahadevan91}
Sridhar Mahadevan and Jonathan Connell.
\newblock Automatic programming of behavior-based robots using reinforcement
  learning.
\newblock In {\em Proceedings of the Ninth National Conference on Artificial
  Intelligence}, Anaheim, CA, 1991.

\bibitem{Mahadevan91a}
Sridhar Mahadevan and Jonathan Connell.
\newblock Scaling reinforcement learning to robotics by exploiting the
  subsumption architecture.
\newblock In {\em Proceedings of the Eighth International Workshop on Machine
  Learning}, pages 328--332, 1991.

\bibitem{Mataric94}
Maja~J. Mataric.
\newblock Reward functions for accelerated learning.
\newblock In W.~W. Cohen and H.~Hirsh, editors, {\em Proceedings of the
  Eleventh International Conference on Machine Learning}. Morgan Kaufmann,
  1994.

\bibitem{McCallum95b}
Andrew~Kachites McCallum.
\newblock {\em Reinforcement Learning with Selective Perception and Hidden
  State}.
\newblock PhD thesis, Department of Computer Science, University of Rochester,
  December 1995.

\bibitem{McCallum93}
R.~Andrew McCallum.
\newblock Overcoming incomplete perception with utile distinction memory.
\newblock In {\em Proceedings of the Tenth International Conference on Machine
  Learning}, pages 190--196, Amherst, Massachusetts, 1993. Morgan Kaufmann.

\bibitem{McCallum95}
R.~Andrew McCallum.
\newblock Instance-based utile distinctions for reinforcement learning with
  hidden state.
\newblock In {\em Proceedings of the Twelfth International Conference Machine
  Learning}, pages 387--395, San Francisco, CA, 1995. Morgan Kaufmann.

\bibitem{Meeden93}
Lisa Meeden, G.~McGraw, and D.~Blank.
\newblock Emergent control and planning in an autonomous vehicle.
\newblock In D.S. Touretsky, editor, {\em Proceedings of the Fifteenth Annual
  Meeting of the Cognitive Science Society}, pages 735--740. Lawerence Erlbaum
  Associates, Hillsdale, NJ, 1993.

\bibitem{Millan96}
Jose del~R. Millan.
\newblock Rapid, safe, and incremental learning of navigation strategies.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics}, 26(3),
  1996.

\bibitem{Monahan82}
George~E. Monahan.
\newblock A survey of partially observable {M}arkov decision processes:
  {T}heory, models, and algorithms.
\newblock {\em Management Science}, 28:1--16, January 1982.

\bibitem{Moore91}
Andrew~W. Moore.
\newblock Variable resolution dynamic programming: {E}fficiently learning
  action maps in multivariate real-valued spaces.
\newblock In {\em Proc. Eighth International Machine Learning Workshop}, 1991.

\bibitem{Moore94}
Andrew~W. Moore.
\newblock The parti-game algorithm for variable resolution reinforcement
  learning in multidimensional state-spaces.
\newblock In J.~D. Cowan, G.~Tesauro, and J.~Alspector, editors, {\em Advances
  in Neural Information Processing Systems 6}, pages 711--718, San Mateo, CA,
  1994. Morgan Kaufmann.

\bibitem{Moore92}
Andrew~W. Moore and Christopher~G. Atkeson.
\newblock An investigation of memory-based function approximators for learning
  control.
\newblock Technical report, {MIT} Artifical Intelligence Laboratory, Cambridge,
  MA, 1992.

\bibitem{Moore93b}
Andrew~W. Moore and Christopher~G. Atkeson.
\newblock Prioritized sweeping: {R}einforcement learning with less data and
  less real time.
\newblock {\em Machine Learning}, 13, 1993.

\bibitem{MooreAtkesonSchaal95}
Andrew~W. Moore, Christopher~G. Atkeson, and S.~Schaal.
\newblock Memory-based learning for control.
\newblock Technical Report CMU-RI-TR-95-18, CMU Robotics Institute, 1995.

\bibitem{Narendra89}
Kumpati Narendra and M.~A.~L. Thathachar.
\newblock {\em Learning Automata: {A}n Introduction}.
\newblock Prentice-Hall, Englewood Cliffs, NJ, 1989.

\bibitem{Narendra74}
Kumpati~S. Narendra and M.~A.~L. Thathachar.
\newblock Learning automata---a survey.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
  4(4):323--334, July 1974.

\bibitem{Peng93}
Jing Peng and Ronald~J. Williams.
\newblock Efficient learning and planning within the {D}yna framework.
\newblock {\em Adaptive Behavior}, 1(4):437--454, 1993.

\bibitem{Peng94}
Jing Peng and Ronald~J. Williams.
\newblock Incremental multi-step {Q}-learning.
\newblock In {\em Proceedings of the Eleventh International Conference on
  Machine Learning}, pages 226--232, San Francisco, CA, 1994. Morgan Kaufmann.

\bibitem{Pomerleau93}
Dean~A. Pomerleau.
\newblock {\em Neural network perception for mobile robot guidance}.
\newblock Kluwer Academic Publishing, 1993.

\bibitem{Puterman94}
Martin~L. Puterman.
\newblock {\em Markov Decision Processes---Discrete Stochastic Dynamic
  Programming}.
\newblock John Wiley \& Sons, Inc., New York, NY, 1994.

\bibitem{Puterman78}
Martin~L. Puterman and Moon~Chirl Shin.
\newblock Modified policy iteration algorithms for discounted {M}arkov decision
  processes.
\newblock {\em Management Science}, 24:1127--1137, 1978.

\bibitem{Ring94}
M.~B. Ring.
\newblock {\em Continual Learning in Reinforcement Environments}.
\newblock PhD thesis, University of Texas at Austin, Austin, Texas, August
  1994.

\bibitem{Rude93}
Ulrich R{\"u}de.
\newblock {\em Mathematical and computational techniques for multilevel
  adaptive methods}.
\newblock Society for Industrial and Applied Mathematics, Philadelphia,
  Pennsylvania, 1993.

\bibitem{Rumelhart86b}
D.~E. Rumelhart and J.~L. McClelland, editors.
\newblock {\em Parallel Distributed Processing: {E}xplorations in the
  microstructures of cognition. Volume 1: {F}oundations}.
\newblock The MIT Press, Cambridge, MA, 1986.

\bibitem{rmmr:onln}
G.~A. Rummery and M.~Niranjan.
\newblock {On-line Q-learning using connectionist systems}.
\newblock Technical Report {CUED/F-INFENG/TR166}, {Cambridge University}, 1994.

\bibitem{Rust96}
John Rust.
\newblock Numerical dynamic programming in economics.
\newblock In {\em Handbook of Computational Economics}. Elsevier, North
  Holland, 1996.

\bibitem{Sage77}
A.~P. Sage and C.~C. White.
\newblock {\em Optimum Systems Control}.
\newblock Prentice Hall, 1977.

\bibitem{Salganicoff95}
Marcos Salganicoff and Lyle~H. Ungar.
\newblock Active exploration and learning in real-valued spaces using
  multi-armed bandit allocation indices.
\newblock In Armand Prieditis and Stuart Russell, editors, {\em Proceedings of
  the Twelfth International Conference on Machine Learning}, pages 480--487,
  San Francisco, CA, 1995. Morgan Kaufmann.

\bibitem{Samuel59}
A.~L. Samuel.
\newblock Some studies in machine learning using the game of checkers.
\newblock {\em IBM Journal of Research and Development}, 3:211--229, 1959.
\newblock Reprinted in E. A. Feigenbaum and J. Feldman, editors, {\it Computers
  and Thought}, McGraw-Hill, New York 1963.

\bibitem{Schaal94b}
S.~Schaal and Christopher Atkeson.
\newblock Robot juggling: {A}n implementation of memory-based learning.
\newblock {\em Control Systems Magazine}, 14, 1994.

\bibitem{Schmidhuber:96ec}
J.~Schmidhuber.
\newblock A general method for multi-agent learning and incremental
  self-improvement in unrestricted environments.
\newblock In X.~Yao, editor, {\em Evolutionary Computation: Theory and
  Applications}. Scientific Publ. Co., Singapore, 1996.

\bibitem{Schmidhuber:91singaporecur}
J.~H. Schmidhuber.
\newblock Curious model-building control systems.
\newblock In {\em Proc. International Joint Conference on Neural Networks,
  Singapore}, volume~2, pages 1458--1463. IEEE, 1991.

\bibitem{Schmidhuber91}
J\"{u}rgen~H. Schmidhuber.
\newblock Reinforcement learning in {M}arkovian and non-{M}arkovian
  environments.
\newblock In D.~S. Lippman, J.~E. Moody, and D.~S. Touretzky, editors, {\em
  Advances in Neural Information Processing Systems 3}, pages 500--506, San
  Mateo, CA, 1991. Morgan Kaufmann.

\bibitem{Schraudolph94}
Nicol~N. Schraudolph, Peter Dayan, and Terrence~J. Sejnowski.
\newblock Temporal difference learning of position evaluation in the game of
  {G}o.
\newblock In J.~D. Cowan, G.~Tesauro, and J.~Alspector, editors, {\em Advances
  in Neural Information Processing Systems 6}, pages 817--824, San Mateo, CA,
  1994. Morgan Kaufmann.

\bibitem{Schrijver86}
Alexander Schrijver.
\newblock {\em Theory of Linear and Integer Programming}.
\newblock Wiley-Interscience, New York, NY, 1986.

\bibitem{Schwartz93}
Anton Schwartz.
\newblock A reinforcement learning method for maximizing undiscounted rewards.
\newblock In {\em Proceedings of the Tenth International Conference on Machine
  Learning}, pages 298--305, Amherst, Massachusetts, 1993. Morgan Kaufmann.

\bibitem{Singh94c}
Satinder~P. Singh, Andrew~G. Barto, Roderic Grupen, and Christopher Connolly.
\newblock Robust reinforcement learning in motion planning.
\newblock In J.~D. Cowan, G.~Tesauro, and J.~Alspector, editors, {\em Advances
  in Neural Information Processing Systems 6}, pages 655--662, San Mateo, CA,
  1994. Morgan Kaufmann.

\bibitem{Singh96}
Satinder~P. Singh and Richard~S. Sutton.
\newblock Reinforcement learning with replacing eligibility traces.
\newblock {\em Machine Learning}, 22(1), 1996.

\bibitem{Singh92c}
Satinder~Pal Singh.
\newblock Reinforcement learning with a hierarchy of abstract models.
\newblock In {\em Proceedings of the Tenth National Conference on Artificial
  Intelligence}, pages 202--207, San Jose, CA, 1992. AAAI Press.

\bibitem{Singh92}
Satinder~Pal Singh.
\newblock Transfer of learning by composing solutions of elemental sequential
  tasks.
\newblock {\em Machine Learning}, 8(3):323--340, 1992.

\bibitem{Singh93}
Satinder~Pal Singh.
\newblock {\em Learning to Solve {M}arkovian Decision Processes}.
\newblock PhD thesis, Department of Computer Science, University of
  Massachusetts, 1993.
\newblock Also, CMPSCI Technical Report 93-77.

\bibitem{stng:stch}
Robert~F. Stengel.
\newblock {\em Stochastic Optimal Control}.
\newblock John Wiley and Sons, 1986.

\bibitem{Sutton96}
R.~S. Sutton.
\newblock {Generalization in Reinforcement Learning: Successful Examples Using
  Sparse Coarse Coding}.
\newblock In D.~Touretzky, M.~Mozer, and M.~Hasselmo, editors, {\em {Neural
  Information Processing Systems 8}}, 1996.

\bibitem{Sutton84}
Richard~S. Sutton.
\newblock {\em Temporal Credit Assignment in Reinforcement Learning}.
\newblock PhD thesis, University of Massachusetts, Amherst, MA, 1984.

\bibitem{Sutton88}
Richard~S. Sutton.
\newblock Learning to predict by the method of temporal differences.
\newblock {\em Machine Learning}, 3(1):9--44, 1988.

\bibitem{Sutton90}
Richard~S. Sutton.
\newblock Integrated architectures for learning, planning, and reacting based
  on approximating dynamic programming.
\newblock In {\em Proceedings of the Seventh International Conference on
  Machine Learning}, Austin, TX, 1990. Morgan Kaufmann.

\bibitem{Sutton91}
Richard~S. Sutton.
\newblock Planning by incremental dynamic programming.
\newblock In {\em Proceedings of the Eighth International Workshop on Machine
  Learning}, pages 353--357. Morgan Kaufmann, 1991.

\bibitem{Tesauro92b}
Gerald Tesauro.
\newblock Practical issues in temporal difference learning.
\newblock {\em Machine Learning}, 8:257--277, 1992.

\bibitem{Tesauro94}
Gerald Tesauro.
\newblock {TD-Gammon}, a self-teaching backgammon program, achieves
  master-level play.
\newblock {\em Neural Computation}, 6(2):215--219, 1994.

\bibitem{Tesauro95}
Gerald Tesauro.
\newblock Temporal difference learning and {TD}-{G}ammon.
\newblock {\em Communications of the ACM}, 38(3):58--67, March 1995.

\bibitem{Tham94}
C-K. Tham and R.~W. Prager.
\newblock A modular q-learning architecture for manipulator task decomposition.
\newblock In {\em Proceedings of the Eleventh International Conference on
  Machine Learning}, San Francisco, CA, 1994. Morgan Kaufmann.

\bibitem{Thrun94}
Sebastian Thrun.
\newblock Learning to play the game of chess.
\newblock In G.~Tesauro, D.~S. Touretzky, and T.~K. Leen, editors, {\em
  Advances in Neural Information Processing Systems 7}, Cambridge, MA, 1995.
  The MIT Press.

\bibitem{Thrun93b}
Sebastian Thrun and Anton Schwartz.
\newblock Issues in using function approximation for reinforcement learning.
\newblock In M.~Mozer, P.~Smolensky, D.~Touretzky, J.~Elman, and A.~Weigend,
  editors, {\em Proceedings of the 1993 Connectionist Models Summer School},
  Hillsdale, NJ, 1993. Lawrence Erlbaum.

\bibitem{Thrun92}
Sebastian~B. Thrun.
\newblock The role of exploration in learning control.
\newblock In David~A. White and Donald~A. Sofge, editors, {\em Handbook of
  Intelligent Control: {N}eural, Fuzzy, and Adaptive Approaches}. Van Nostrand
  Reinhold, New York, NY, 1992.

\bibitem{Tsitsiklis94}
John~N. Tsitsiklis.
\newblock Asynchronous stochastic approximation and {Q}-learning.
\newblock {\em Machine Learning}, 16(3), September 1994.

\bibitem{Tsitsiklis96}
John~N. Tsitsiklis and Ben Van~Roy.
\newblock Feature-based methods for large scale dynamic programming.
\newblock {\em Machine Learning}, 22(1), 1996.

\bibitem{Valiant84}
L.~G. Valiant.
\newblock A theory of the learnable.
\newblock {\em Communications of the {ACM}}, 27(11):1134--1142, November 1984.

\bibitem{Watkins89}
Christopher J. C.~H. Watkins.
\newblock {\em Learning from Delayed Rewards}.
\newblock PhD thesis, King's College, Cambridge, UK, 1989.

\bibitem{Watkins92}
Christopher J. C.~H. Watkins and Peter Dayan.
\newblock Q-learning.
\newblock {\em Machine Learning}, 8(3):279--292, 1992.

\bibitem{Whitehead91b}
Steven~D. Whitehead.
\newblock Complexity and cooperation in {Q}-learning.
\newblock In {\em Proceedings of the Eighth International Workshop on Machine
  Learning}, Evanston, IL, 1991. Morgan Kaufmann.

\bibitem{Williams87}
Ronald~J. Williams.
\newblock A class of gradient-estimating algorithms for reinforcement learning
  in neural networks.
\newblock In {\em Proceedings of the IEEE First International Conference on
  Neural Networks}, San Diego, CA, 1987.

\bibitem{Williams92}
Ronald~J. Williams.
\newblock Simple statistical gradient-following algorithms for connectionist
  reinforcement learning.
\newblock {\em Machine Learning}, 8(3):229--256, 1992.

\bibitem{Williams93b}
Ronald~J. Williams and Leemon~C. Baird, III.
\newblock Analysis of some incremental variants of policy iteration: {F}irst
  steps toward understanding actor-critic learning systems.
\newblock Technical Report NU-CCS-93-11, Northeastern University, College of
  Computer Science, Boston, MA, September 1993.

\bibitem{Williams93}
Ronald~J. Williams and Leemon~C. Baird, III.
\newblock Tight performance bounds on greedy policies based on imperfect value
  functions.
\newblock Technical Report NU-CCS-93-14, Northeastern University, College of
  Computer Science, Boston, MA, November 1993.

\bibitem{Wilson95}
Stewart Wilson.
\newblock Classifier fitness based on accuracy.
\newblock {\em Evolutionary Computation}, 3(2):147--173, 1995.

\bibitem{Zhang95}
W.~Zhang and T.~G. Dietterich.
\newblock A reinforcement learning approach to job-shop scheduling.
\newblock In {\em Proceedings of the International Joint Conference on
  Artificial Intellience}, 1995.

\end{thebibliography}
