/*
 * MiningMart Version 1.0
 * 
 * Copyright (C) 2006 Martin Scholz, Timm Euler, 
 *                    Daniel Hakenjos, Katharina Morik
 *
 * Contact: miningmart@ls8.cs.uni-dortmund.de
 *
 * A list of contributing developers (other than the copyright 
 * holders) can be found at
 * http://mmart.cs.uni-dortmund.de/downloads/download.html
 * 
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program, see the file MM_HOME/LICENSE; if not, write
 * to the Free Software Foundation, Inc., 51 Franklin Street, Fifth
 * Floor, Boston, MA 02110-1301, USA.
 */
package edu.udo.cs.miningmart.compiler;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.sql.SQLException;
import java.util.Calendar;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.logging.Level;

import edu.udo.cs.miningmart.db.CompilerDatabaseService;
import edu.udo.cs.miningmart.db.ConfigReader;
import edu.udo.cs.miningmart.db.DB;
import edu.udo.cs.miningmart.exception.DbConnectionClosed;
import edu.udo.cs.miningmart.exception.M4CompilerError;
import edu.udo.cs.miningmart.exception.M4CompilerWarning;
import edu.udo.cs.miningmart.exception.M4Exception;
import edu.udo.cs.miningmart.exception.OperatorRuntimeConditionViolated;
import edu.udo.cs.miningmart.exception.UserError;
import edu.udo.cs.miningmart.m4.Case;
import edu.udo.cs.miningmart.m4.Column;
import edu.udo.cs.miningmart.m4.Columnset;
import edu.udo.cs.miningmart.m4.Concept;
import edu.udo.cs.miningmart.m4.M4InterfaceContext;
import edu.udo.cs.miningmart.m4.M4Object;
import edu.udo.cs.miningmart.m4.Operator;
import edu.udo.cs.miningmart.m4.Step;
import edu.udo.cs.miningmart.m4.utils.Print;
import edu.udo.cs.miningmart.operator.ExecutableOperator;

/**
 * Objects of this class realize a single thread executing a command to the compiler.
 * This is used by the RMI interface class to create different instances for concurrent 
 * compiler instances, but also by the class
 * <code>miningmart.compiler.CreateCompiler</code> to run a single local (non-RMI)
 * instance for a single compiliation task (e.g. command line based).
 * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl
 * @see miningmart.compiler.CreateCompiler
 * 
 * @author Martin Scholz
 * @version $Id: CompilerAccessLogic.java,v 1.11 2006/04/11 14:10:09 euler Exp $
 */
public class CompilerAccessLogic implements M4InterfaceContext, CompilerAccess
{
	// One static instance of thread control, to sequentialize critical accesses
	// to the database:
	private static final CompilerThreadControl ctc = new CompilerThreadControl();

	// After how many compiled steps should the database connection be refreshed?
	// Prevents us from too many open cursors!
	private static final int MAX_STEPS_PER_CONNECTION = 150;
	private int stepsDuringDbConnection = 0;
	
	// If set to true, the Thread will be stopped by an exception very soon.
	private boolean stopThread = false;

	/**
	 * Each instance of CompilerAccessLogic has its own instance of
	 * <code>DB</code>, because it needs its own cache, etc.
	 * */
    private final CompilerDatabaseService m4db;

	/**
	 * If for some reason it is impossible to write to a <code>Case</code>
	 * dependent logfile, or if there is no case information available (yet),
	 * use this default <code>Print</code> object.
	 */    
    private Print defaultPrintObject;
    
	/** 
	 * Each instance of CompilerAccessLogic has its own instance of class
	 * <code>Print</code>, so there is a separate logfile for each case.
	 */    
    private Print casePrintObject;
    
    /**
     * Except for the default <code>CompilerAccessLogic</code> object created by
     * the compiler interface each <code>CompilerAccessLogic</code> corresponds to
     * a single <code>Case</code>, represented by the object stored here.
     */
    private Case  caseObj;

	/**
	 * This field represents the verbosity level of the <code>Case/code>-dependent
	 * <code>Print</code> object used by this <code>CompilerAccessLogic</code>.
	 * @see miningmart.compiler.utils.Print
	 */	
	private Level verbosity;


	// -------- Constructors --------

	/**
	 * This constructor takes all information needed from an existing
	 * database connection <b>and shares the cache with the <code>DB</code>
	 * object</b>!
	 * 
	 * @param db an existing database connection object of type <code>DB</code>
	 * @param printObject the Print object to be used for log messages
	 */
	public CompilerAccessLogic(DB db, Print printObject) {
		if (db instanceof CompilerDatabaseService) {
			this.m4db = (CompilerDatabaseService) db;
		}
		else {
			this.m4db = new CompilerDatabaseService(db, this);
		}
		String v = System.getProperty(edu.udo.cs.miningmart.m4.M4Interface.SYSTEM_PROP_PRINT_VERBOSITY);
		if (v != null) {
			this.verbosity = Print.getVerbosityLevelForName(v);
		}
		else {
			this.verbosity = Print.DEFAULT_VERBOSITY;
		}
		this.defaultPrintObject = printObject;
	}

	/**
	 * Sets up a new instance of <code>CompilerAccessLogic</code>, using the
	 * default verbosity and the specified configuration file.
	 * 
	 * @param myDbConfigFile the path to the configuration file describing the
	 * 		  settings of the database 
	 * @param printObject the Print object to be used for log messages
	 * 
	 * @see miningmart.compiler.utils.Print
	 */
    public CompilerAccessLogic(String myDbConfigFile)
    	throws IOException, SQLException
    {
    	this(myDbConfigFile, null);
    }

	/**
	 * Sets up a new instance of <code>CompilerAccessLogic</code>, using the
	 * specified verbosity and configuration file.
	 * 
	 * @param myDbConfigFile the path to the configuration file describing the
	 * 		  settings of the database 
	 * @param verbosity the verbosity of the <code>Print</code> object associated
	 * 		  with <code>this</code> object.
	 * @param printObject the Print object to be used for log messages
	 * 
	 * @see miningmart.compiler.utils.Print
	 */
    public CompilerAccessLogic(String myDbConfigFile, Level verbosity)
    	throws IOException, SQLException
    {
		super();

		if (verbosity == null || verbosity.intValue() < 0) {
			this.verbosity = null;
		} 
		else {
			this.verbosity = verbosity;
		}
		this.initDefaultPrintObject();

		ConfigReader cr = null;
		try {
			cr = new ConfigReader(myDbConfigFile);
		}
		catch (M4Exception m4e) {
			throw new IOException(m4e.getMessage());
		}
		this.m4db = new CompilerDatabaseService(cr, true, this);	    
    }


	// ----------------

	/**
	 * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#compileStep(long, boolean)
	 */
    public void compileStep(long stepId, boolean lazyMode)
		throws M4CompilerInterfaceError, M4CompilerWarning, UserError
	{
		long caseId = this.getCaseIdByStepId(stepId);
		this.setCase(caseId);
		
		try {
			String warningMessage = "";
			final Step step = (Step) this.getM4db().getM4Object(stepId, Step.class);
			// Vector sequence = new Vector();

			Step pred;
			do {
				pred = this.findUncompiledPred(step);
				try {
					this.compileStepSub(pred, lazyMode);
				}
				catch (M4CompilerWarning mw) {
					warningMessage += mw.getMessage() + "\n";
				}
			} while ( ! step.equals(pred) );
			if ( ! warningMessage.equals("")) {
				throw new M4CompilerWarning(warningMessage);
			}
		}
		catch(OperatorRuntimeConditionViolated e) {
			throw e; // hide from subsequent catch clause
		}
		catch (M4Exception e)
		{   throw new M4CompilerInterfaceError(e.getMessage());   }
    }

	/** 
	 * Checks all predecessor Steps, if they need to be compiled before
	 * and returns one that can be compiled directly. Finally the specified
	 * Step will be returned.
	 */
	private Step findUncompiledPred(Step step)
		throws M4Exception
	{ 
		// get predecessors:
		final edu.udo.cs.miningmart.m4.Case m4case = step.getTheCase();
		Iterator pred = m4case.getReverseIterator();
		
		// skip until specified step:
		while (pred.hasNext() && ! step.equals(pred.next()));

		// scan possible predecessors:
		while (pred.hasNext()) {
			Step current = (Step) pred.next();
			if (m4case.containsDependency(current, step) && ! current.isCompiled()) {
				step = current;
			}
		}
		return step;
	}
	
	/** Helper method doing the actual compilation of the specified step. */
	private void compileStepSub(Step step, boolean lazyMode)
		throws M4Exception, M4CompilerWarning, M4CompilerInterfaceError
	{
		this.deleteTrashBeforeCompilingStep(step);

		try {
			this.compileStepMultiStepControl(step, lazyMode, false);
		}
		catch(OperatorRuntimeConditionViolated e) {
			throw e; // hide from subsequent catch clause
		}
		catch (M4Exception e)
		{   throw new M4CompilerInterfaceError(e.getMessage());   }

		this.printStepExecInfo(step, lazyMode);		
	}


	/**
	 * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#compileStep(String, boolean)
	 */
    public void compileStep(String stepName, boolean lazyMode)
		throws M4CompilerInterfaceError, M4CompilerWarning, UserError
	{
		long stepId = findStepId(stepName);
		this.compileStep(stepId, lazyMode);
    }


    /**
     * This method implements the control structure. If multiple
     * concepts have multiple column sets then the operator is run for
     * all elements of the cartesian product of column sets!
     */
    private void compileStepMultiStepControl(Step step, boolean lazyMode, boolean computeStatistics)
		throws M4CompilerInterfaceError, M4CompilerWarning, M4Exception
	{		
		try {
			// Find all input concepts of the given step:
			Collection inputConcepts = this.getDb().getInputConceptsFor(step);		
			Iterator it = inputConcepts.iterator();
			int minColSetNumber = Integer.MAX_VALUE;
			while (it.hasNext()) {
				Concept c =	(Concept) it.next();
				// Collect and initialize those concepts having more than 1 column set:
				Collection colsets = c.getColumnSets();
				if (colsets == null || colsets.size() == 0) {
					throw new M4CompilerError(
						"A parameter of type Concept ('" + c.getName() + "') of step '"
						+ step.getName() + "' has no Columnset!");
				}
				int csNum = colsets.size();
				if (csNum < minColSetNumber) {
					minColSetNumber = csNum;
				}
				if (csNum > 1) {
					c.initMultiStepSupport(colsets);
				}
			}
			// If not all concepts have multiple column sets then no extra control is necessary.
			if (minColSetNumber <= 1) {
				compileStepBody(step, lazyMode);
			}
			/* If the step should be compiled in lazy mode then forget
			 * about switching columnsets! But BaseAttribute templates
			 * need to be initialized if we have multi columnset
			 * concepts! */
			else if (lazyMode) {
				compileStepBody(step, true);
			}
			else {
				// Otherwise execute the step for the valid combinations of column sets.
				// This means additional effort for those concepts only, where there are
				// multiple columnsets present. For other concepts the single column set
				// will be loaded correctly by the operator's usual mechanism!
				do {
					compileStepBody(step, lazyMode);
				}
				while (nextCSSetting(inputConcepts));
			}
		}
		catch (SQLException ex) {
			//    Print.doPrint(ex);
			throw new M4CompilerInterfaceError(
				"SQLException trying to compile step " + step.getId()
				+ " in " + (lazyMode ? "lazy" : "eager")
				+ " mode:\n" + ex.getMessage() + "\nCompilation stopped.");
		}
		catch (M4CompilerError ex) {
			throw new M4CompilerInterfaceError(
				"M4CompilerError trying to compile step " + step.getId()
				+ " in " + (lazyMode ? "lazy" : "eager")
				+ " mode:\n" + ex.getMessage() + "\nCompilation stopped.");
		}
	}
	
    /**
     * Helper method: Chooses the next combination of column sets and
     * switches the features accordingly!
     * @return <code>true</code> iff the combination is valid and the step
     * is ready to be executed.
     */
    private boolean nextCSSetting(Collection inputConcepts)
		throws M4CompilerError, UserError
	{
		Iterator it = inputConcepts.iterator();

		while (it.hasNext()) {
			Concept c = (Concept) it.next();
			try {
				if (c.hasNextColumnSet()) {
				    c.getNextColumnSet();
				}
				else { // Reset all Concepts. Should be superfluous.
					it = inputConcepts.iterator();
					while (it.hasNext()) {
						((Concept) it.next()).resetColumnSets();
					}
			    	return (false);
				}
			}
			catch(OperatorRuntimeConditionViolated e) {
				throw e; // hide from subsequent catch clause
			}
			catch (M4Exception m4e) {
				throw new M4CompilerError("M4 interface error in CompilerAccessLogic: " + m4e.getMessage());
			}				
		}
		return true;
	}

    /* -------------------------------------------------------------------*/

	/*
	 * This method just switches off statistics before compiling a <code>Step</code>.
	 * @see edu.udo.cs.miningmart.compiler.CompilerAccessLogic#compileStepBody(Step, boolean)
	 * @param stepId the step id
	 * @param lazyMode compile in lazy mode or not?
	 * @param computeStatistics compute statistics or not
	 *
    private void compileStepBody
    	(long stepId, boolean lazyMode, boolean computeStatistics)
		throws M4CompilerInterfaceError, UserError
	{
	    try {
			Step step = (Step) this.getDb().getM4Object(stepId, Step.class);
			compileStepBody(step, lazyMode);
	    }
	    catch (M4CompilerError ex) {
			this.doPrint(ex);
			throw new M4CompilerInterfaceError
			    ("An M4CompilerError occured executing step no." + stepId
		    	 + " in " + (lazyMode ? "lazy" : "eager") + " Mode:\n"
			     + ex.getMessage());
	    }
	    catch (SQLException ex) {
			this.doPrint(ex);
			throw new M4CompilerInterfaceError
			    ("An SQLException occured executing step no." + stepId
		    	 + " in " + (lazyMode ? "lazy" : "eager") + " Mode:\n"
			     + ex.getMessage());
	    }
		catch(OperatorRuntimeConditionViolated e) {
			throw e; // hide from subsequent catch clause
		}
		catch (M4Exception m4e)
		{   throw new M4CompilerInterfaceError("M4 interface error in CompilerAccessLogic: " + m4e.getMessage());  }
	}
	*/

	/**
	 * This method realizes the single step compiler and must only be used
	 * by the <code>private</code> method
	 * <code>compileStepMultiStepControl(Step, boolean)</code>
	 * 
	 * @param step the <code>Step</code> to be compiled
	 * @param lazyMode indicates if this <code>Step</code> should be compiled
	 * in the less time consuming lazy mode.
	 * @throws UserError
	 * @throws M4CompilerError
	 * @throws SQLException
	 * @throws M4Exception
	 */
    private void compileStepBody(Step step, boolean lazyMode)
		throws UserError, M4CompilerError, M4CompilerWarning, M4Exception, SQLException
	{
		this.doPrint(Print.COMPILER_STEP_CONTROL, "Waiting for resources to compile step " + step.getId() +".");
		ctc.addThread(CompilerThreadControl.STEP_COMPILATION);
		boolean success = false;
		try {					   
			if (this.stepsDuringDbConnection++ >= MAX_STEPS_PER_CONNECTION)
			{
				try {
					this.getDb().getFreshM4Connection();
					this.stepsDuringDbConnection = 0;
				}
				catch (SQLException sqle) {
					this.doPrint(Print.COMPILER_CASE_CONTROL, "CompilerAccessLogic: Could not refresh DB connection!\n" +sqle);
				}
			}

			// printBeforeStepInfo(step.getId(), lazyMode);
			ExecutableOperator exOp = getExecutableOperator(step.getTheOperator());
			this.doPrint(Print.COMPILER_STEP_CONTROL, "Executing step '" + step.getName() +"' (ID: " + step.getId() + ").");
			exOp.load(step);
			String warningMessage = null;
			try {
				exOp.execute(lazyMode);
			}
			catch (M4CompilerWarning mcw) {
				warningMessage = mcw.getMessage();
			}
			// Set the 'Step compiled' flag in M4Trash:
			try {
				step.setCompiled();

				// Commit the changes to the database.
				this.getDb().updateDatabase();

				// Store the success for the User Interaction.
				success = true;
			}
			catch (M4Exception e) {
				throw new M4CompilerError(
					"Error when trying to set the compiled flag for step "
					+ step.getId() + ":\n" + e.getMessage());
			}
			if (warningMessage != null) {
				throw new M4CompilerWarning(warningMessage);
			}
		}
		finally { // Do not block other threads in case of errors!
			ctc.subThread(CompilerThreadControl.STEP_COMPILATION);
			this.doPrint(Print.COMPILER_STEP_CONTROL,
					"Compilation of step '" + step.getName()
					+ (success ? "' done." : "' was not successful!"));
		}
    }

	// Helper field of getExecutableOperator(String opClass):
	private static final Class[] EXEC_OP_CONSTRUCTOR_TYPE = { }; // constructors do not expect any parameters

	/**
	 * This method bridges the gap between the M4 operator objects and the
	 * executable operators. It reads an operator class name, as delivered e.g.
	 * by <code>Step.getOperatorClass()</code> and returns an executable operator.
	 * 
	 * @param opClass the operator class as a <code>String</code>
	 * @return an instantiated <code>ExecutableOperator</code>
	 */
	public static ExecutableOperator getExecutableOperator(Operator operator) 
	throws M4CompilerError {
		String opClass = operator.getName();
		try {
			Class theOpClass;
			try {
				theOpClass =
					Class.forName("edu.udo.cs.miningmart.operator." + opClass);
			}
			catch (ClassNotFoundException e1) {
				try {
					theOpClass =
						Class.forName(
							"edu.udo.cs.miningmart.operator.dista." + opClass);
				}
				catch (ClassNotFoundException e2) {
						theOpClass =
							Class.forName(
								"edu.udo.cs.miningmart.operator.uep." + opClass);
				}
			}
			Constructor c = theOpClass.getConstructor(EXEC_OP_CONSTRUCTOR_TYPE);
			Object[] params = new Object[0];
			return (ExecutableOperator) c.newInstance(params);
		}
		catch (Exception e) {
			throw new M4CompilerError(
				"CompilerAccessLogic.getExecutableOperator(): Could not find and initialise class "
				+ opClass + "!\n" + e.getMessage());
		}
	}	

	// ---------------------------------------------------------------------

	/**
	 * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#compileAll(long, boolean)
	 */
    public void compileAll(long caseId, boolean lazyMode)
	throws M4CompilerInterfaceError, M4CompilerWarning, UserError
    {
		try {
			String warningMessage = "";
			this.setCase(caseId);
			Iterator steps = this.getCase().getStepIterator();
			final HashSet garbageCollected = new HashSet();
			while (steps.hasNext()) {
				final Step currentStep = (Step) steps.next();
				if (! garbageCollected.contains(currentStep)) {
					this.deleteTrashBeforeCompilingStep(currentStep);
					Collection col = this.getCase().getDependentStepsFor(currentStep);
					garbageCollected.addAll(col);
				} 
	        	this.printBeforeStepInfo(currentStep, lazyMode);
	        	try {
	        		compileStepMultiStepControl(currentStep, lazyMode, false);
	        	}
	        	catch (M4CompilerWarning mw) {
	        		warningMessage += mw.getMessage() + "\n";
	        	}
    		    this.printStepExecInfo(currentStep, lazyMode);
			}
			if ( ! warningMessage.equals("")) {
				throw new M4CompilerWarning(warningMessage);
			}
		}
		catch(OperatorRuntimeConditionViolated e) {
			throw e; // hide from subsequent catch clause
		}
		catch (M4Exception m4e) {
			throw new M4CompilerInterfaceError(
				"M4 interface error in CompilerAccessLogic: "
				+ m4e.getMessage());
		}
    }

	/**
	 * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#compileStepFrom(long, boolean)
	 */
    public void compileStepFrom(long stepId, boolean lazyMode)
		throws M4CompilerInterfaceError, M4CompilerWarning, UserError
    {
		try {
			String warningMessage = "";
			long caseId = this.getCaseIdByStepId(stepId);
			this.setCase(caseId);
			boolean garbageCollected = false;
			Step step = (Step) this.getM4db().getM4Object(stepId, Step.class);
			Iterator steps = this.getCase().getDependentStepsFor(step).iterator();
			while (steps.hasNext()) {
				final Step currentStep = (Step) steps.next();
					
				// Garbage Collection needs to be done just once!
				// The first step after sequentializing deletes the
				// garbage of all the subsequent dependent steps.
				if (garbageCollected == false) {
					this.deleteTrashBeforeCompilingStep(currentStep);
					garbageCollected = true;
				} 
							
	        	this.printBeforeStepInfo(currentStep, lazyMode);
	        	try {
	        		compileStepMultiStepControl(currentStep, lazyMode, false);
	        	}
	        	catch (M4CompilerWarning mcw) {
	        		warningMessage += mcw.getMessage() + "\n";
	        	}
    		    this.printStepExecInfo(currentStep, lazyMode);
			}
			if ( ! warningMessage.equals("")) {
				throw new M4CompilerWarning(warningMessage);
			}
		}
		catch(OperatorRuntimeConditionViolated e) {
			throw e; // hide from subsequent catch clause
		}
		catch (M4Exception m4e) {
			throw new M4CompilerInterfaceError(
				"M4 interface error in CompilerAccessLogic: "
				+ m4e.getMessage());
		}
    }

	/**
	 * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#compileStepTo(long, boolean)
	 */
    public void compileStepTo(long stepId, boolean lazyMode, boolean ignoreCompiledStatus)
		throws M4CompilerInterfaceError, M4CompilerWarning, UserError
    {
		try {
			String warningMessage = "";
			long caseId = this.getCaseIdByStepId(stepId);
			this.setCase(caseId);
			Step step = (Step) this.getM4db().getM4Object(stepId, Step.class);
			
			// first, compile all steps that the given step is dependent on:
			Iterator steps = this.getCase().getStepsToCompileBefore(step, ignoreCompiledStatus).iterator();
			while (steps.hasNext()) {
				final Step currentStep = (Step) steps.next();
	        	this.printBeforeStepInfo(currentStep, lazyMode);
	        	// we may already have deleted the trash for this step, but 
	        	// we don't know it:
	        	this.deleteTrashBeforeCompilingStep(currentStep);
				try {
					compileStepMultiStepControl(currentStep, lazyMode, false);
				}
				catch (M4CompilerWarning mw) {
					warningMessage += mw.getMessage() + "\n";
				}
    		    this.printStepExecInfo(currentStep, lazyMode);
			}
			
			// second, compile the given step:						
        	this.printBeforeStepInfo(step, lazyMode);
        	// we may already have deleted the trash for this step, but 
        	// we don't know it:
        	this.deleteTrashBeforeCompilingStep(step);
        	try {
        		compileStepMultiStepControl(step, lazyMode, false);
        	}
        	catch (M4CompilerWarning mw) {
				warningMessage += mw.getMessage() + "\n";
			}
		    this.printStepExecInfo(step, lazyMode);
		    if ( ! warningMessage.equals("")) {
		    	throw new M4CompilerWarning(warningMessage);
		    }
		}
		catch(OperatorRuntimeConditionViolated e) {
			throw e; // hide from subsequent catch clause
		}
		catch (M4Exception m4e) {
			throw new M4CompilerInterfaceError(
				"M4 interface error in CompilerAccessLogic: "
				+ m4e.getMessage());
		}
    }

	/**
	 * Helper method to delete objects for the step stored in the Trash tables
	 * with additional common output before compiling the specified step.
	 */
	private void deleteTrashBeforeCompilingStep(Step step) 
		throws M4CompilerInterfaceError
	{
			this.deleteTrashFromStep(step); // deletes business- and m4-garbage
	    	this.doPrint(Print.COMPILER_STEP_CONTROL,
	    			this.systemTime()
		    		+ " : Preparing execution of step '"
		    		+ step.getName() + "', checking for MultiSteps...");
	}

	/** Helper method to delete objects for the step stored in the Trash tables. */
	private void deleteTrashFromStep(Step step)
		throws M4CompilerInterfaceError
	{
	    this.doPrint(Print.COMPILER_STEP_CONTROL,
	    		this.systemTime()
			    + " Waiting for resource to run garbage collection for step '"
			    + step.getName() + "'.");

		try {
			ctc.addThread(CompilerThreadControl.GARBAGE_COLLECTION);
			
		    this.doPrint(Print.COMPILER_STEP_CONTROL,
		    		this.systemTime()
			    	+ " Running garbage collection for step '"
			    	+ step.getName() + "'.");
				
			Iterator it;
			{
				edu.udo.cs.miningmart.m4.Case theCase = step.getTheCase();
				Collection dependentSteps = theCase.getDependentStepsFor(step);
				LinkedList reverse = new LinkedList();
				it = dependentSteps.iterator();
				while (it.hasNext()) {
					reverse.addFirst(it.next());
				}
				it = reverse.iterator();
			}
			
			while (it.hasNext()) {
				Step aStep = (Step) it.next();
				if (aStep.getName().startsWith("PivotizeLengthBy")) {
					int i = 0;
				}
				aStep.deleteM4Trash();	
			    aStep.deleteDbTrash();
			}
			
			// writes "m4trash updates" to the database and commits changes to
			// both the m4 and business schema:
			this.getM4db().updateDatabase();
		}
		catch (M4CompilerError e) {
			throw new M4CompilerInterfaceError(e.getMessage());
		}
		catch (M4Exception e) {
			throw new M4CompilerInterfaceError(e.getMessage());
		}
		finally {
			// If execution fails: Try not to block the remaining threads!
			try {
				ctc.subThread(CompilerThreadControl.GARBAGE_COLLECTION);
			}
			catch (M4CompilerError e) { this.doPrint(e); }
		}
	}

	/* Helper method to run a step specified by its (unique, see DB) name. */
	private long findStepId(String stepName)
		throws M4CompilerInterfaceError
	{
		String msg = "";
		try {
			String query =
				"SELECT st_id FROM step_t WHERE st_name = '" + stepName + "'";
			Long stepId = this.getDb().executeM4SingleValueSqlReadL(query);
			if (stepId != null) {
				return (stepId.longValue());
			}
		}
		catch (SQLException e) {
			msg = "\nAn SQLException occured:\n" + e.getMessage();
		}
		catch (DbConnectionClosed m) {
			msg= "\nCompiler Thread was killed!\n" + m.getMessage();
		}
		msg = "M4 Database: Step with name '" + stepName + "' not found!" + msg;
		throw new M4CompilerInterfaceError(msg);
	}
    /* Helper methods for logfile outputs: */

	private void printBeforeStepInfo(Step step, boolean lazyMode) {
	    this.doPrint(Print.COMPILER_STEP_CONTROL, "\n--- " + this.systemTime() + " ---");
	    this.doPrint(Print.COMPILER_STEP_CONTROL, "Trying to execute Step '" + step.getName()
	    				+ "' (ID: " + step.getId() + ") in "
	    				+ (lazyMode ? "lazy" : "eager") + " mode ...");
	}

	private void printStepExecInfo(Step step, boolean lazyMode) {
	    this.doPrint(Print.COMPILER_STEP_CONTROL, "\n--- " + this.systemTime() + " ---");
	    this.doPrint(Print.COMPILER_STEP_CONTROL, "Execution for Step " + step.getName() + " without errors");
	    this.doPrint(Print.COMPILER_STEP_CONTROL, "Mode was set to " + (lazyMode ? "lazy" : "eager"));
	}

	/*
	private void printStepExecInfo(String stepName, boolean lazyMode) {
	    this.doPrint(Print.COMPILER_STEP_CONTROL, "\n--- " + this.systemTime() + " ---");
	    this.doPrint(Print.COMPILER_STEP_CONTROL, "Execution for Step " + stepName + " without errors");
	    this.doPrint(Print.COMPILER_STEP_CONTROL, "Mode was set to " + (lazyMode ? "lazy" : "eager"));
	}
	*/

	private String systemTime() {
		Calendar calendar = Calendar.getInstance();
		final String day     = this.fillUpToTwoDigits(calendar.get(Calendar.DAY_OF_MONTH));
		final String month   = this.fillUpToTwoDigits(calendar.get(Calendar.MONTH) + 1);
		final int year       = calendar.get(Calendar.YEAR);
		final String hour    = this.fillUpToTwoDigits(calendar.get(Calendar.HOUR_OF_DAY));
		final String minutes = this.fillUpToTwoDigits(calendar.get(Calendar.MINUTE));
		final String seconds = this.fillUpToTwoDigits(calendar.get(Calendar.SECOND));
		String s = day + "." + month + "." + year + " " + hour + ":" + minutes + ":" + seconds;
		return s;
	}

	private String fillUpToTwoDigits (int number) {
		String s = Integer.toString(number);
		if (s.length() == 1) {
			s = "0" + s;
		}
		return s;
	}

    // ---------------------------------------------------------------------

    /**
     * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#updateColumnStatistics(long)
     */
    public void updateColumnStatistics(long columnID)
		throws M4CompilerInterfaceError
    {
        try {
       		this.doPrint(Print.COMPILER_STEP_CONTROL,
       			"Waiting for resources to updateColumnStatistics of column "
       			+ columnID + ".");
			ctc.addThread(CompilerThreadControl.STEP_COMPILATION);
       		this.doPrint(Print.COMPILER_STEP_CONTROL,
       			"Starting calculations for updating Column statistics (column id: "
       			+ columnID + ").");
       		try {
	            Column myColumn = (Column) this.getM4db().getM4Object(columnID, Column.class);
    	        final edu.udo.cs.miningmart.m4.BaseAttribute ba = myColumn.getTheBaseAttribute();
    	        if (ba == null) {
    	        	throw new M4CompilerInterfaceError
    	        		("updateColumnStatistics: Trying to calculate statistics for Column "
	    	        	+ "not attached to a BaseAttribute!\nColumn ID is: " + columnID);
    	        }
	            myColumn.updateStatistics();
	       		this.doPrint(Print.COMPILER_STEP_CONTROL,
	       			"Updated column statistics of column " + columnID + ".");
       		}
			catch (M4Exception m4e)
			{   throw new M4CompilerInterfaceError("M4 interface error in CompilerAccessLogic: " + m4e.getMessage());  }
       		finally {
				ctc.subThread(CompilerThreadControl.STEP_COMPILATION);       			
       		}
        }
		catch (M4CompilerError ex) {
		    this.doPrint(ex);
	    	throw new M4CompilerInterfaceError
			("M4CompilerError trying to update ColumnStatistics for Column "
			 + columnID + ":\n");
		}
    }

	/**
     * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#updateColumnsetStatistics(long)
     */
    public void updateColumnsetStatistics(long columnsetID)
    	throws M4CompilerInterfaceError
    {
       try {
       		this.doPrint(Print.COMPILER_STEP_CONTROL,
       			"Waiting for resources to updateColumnsetStatistics of columnset "
       			+ columnsetID + ".");
			ctc.addThread(CompilerThreadControl.STEP_COMPILATION);
       		this.doPrint(Print.COMPILER_STEP_CONTROL,
       			"Starting calculations for updating Columnset statistics (columnset id: "
       			+ columnsetID + ").");
       		try {
	            Columnset myCol = (Columnset) this.getDb().getM4Object(columnsetID, Columnset.class);
        	    myCol.updateStatistics();
	       		this.doPrint(Print.COMPILER_STEP_CONTROL,
       			"Updated columnset statistics of columnset "	+ columnsetID + ".");
       		}
			catch (M4Exception m4e) {
				throw new M4CompilerInterfaceError(
					"M4 interface error in CompilerAccessLogic: "
					+ m4e.getMessage());
			}
       		finally {
				ctc.subThread(CompilerThreadControl.STEP_COMPILATION);
       		}
       }
       catch (M4CompilerError ex) {
            this.doPrint(ex);
	    	throw new M4CompilerInterfaceError(
	    		"M4CompilerError trying to update ColumnsetStatistics for ColumnSet "
			    + columnsetID + ":\n");
       }
    }

	/**
     * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#updateStatisticsForConcept(long)
     */
    public void updateStatisticsForConcept(long conceptID)
    	throws M4CompilerInterfaceError
	{
        try {
        	this.doPrint(Print.COMPILER_STEP_CONTROL,
       			"Waiting for resources to updateStatisticsForConcept of concept "
       			+ conceptID + ".");
			ctc.addThread(CompilerThreadControl.STEP_COMPILATION);
       		this.doPrint(Print.COMPILER_STEP_CONTROL,
       			"Starting calculations for updating Concept statistics (Concept id: "
       			+ conceptID + ").");

       		try {      			
		        Concept myConcept = (Concept) this.getM4db().getM4Object(conceptID, Concept.class);
				if (myConcept == null) {
					throw new M4CompilerInterfaceError(
						"CompilerAccessLogic.updateStatisticsForConcept:"
						+ "Concept with ID " + conceptID + " not found!");
				}

    	        Collection cols = myConcept.getColumnSets();
        	    // Update statistics for all Columnsets
        	    Iterator it = cols.iterator();
        	    while (it.hasNext()) {
					Columnset cs = (Columnset) it.next();
					cs.updateStatistics();
				}
				this.getM4db().updateDatabase();
            	this.doPrint(Print.COMPILER_STEP_CONTROL,
       				"Updated statistics for concept " + conceptID + ".");
            	
       		}
			catch (M4Exception m4e) {
				throw new M4CompilerInterfaceError(
					"M4 interface error in CompilerAccessLogic: "
					+ m4e.getMessage());
			}			
            finally {
         		ctc.subThread(CompilerThreadControl.STEP_COMPILATION);  	
            }
        }
		catch (M4CompilerError ex) {
    	    this.doPrint(ex);
	    	throw new M4CompilerInterfaceError
			("M4CompilerError trying to update statistics for concept " + conceptID + ":\n");
        }
    }

	/**
     * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#deleteTrashForStep(long)
     */
	public void deleteTrashForStep(long stepId)
		throws M4CompilerInterfaceError
	{
		if (this.getCase() == null) {
			long caseId = this.getCaseIdByStepId(stepId);
			this.setCase(caseId);
		}
		try {
			Step step = (Step) this.getM4db().getM4Object(stepId, Step.class);
			this.deleteTrashFromStep(step);	
		}
		catch (M4Exception e) {
			throw new M4CompilerInterfaceError(e.getMessage());	
		}
	}



	/**
     * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#deleteTrashForCase(long)
     */
	public void deleteTrashForCase(long caseId)
		throws M4CompilerInterfaceError
	{
		if (this.getCase() == null) {
			this.setCase(caseId);
		}
		HashSet gcDone = new HashSet();
		try {
			Iterator it = this.getCase().getStepIterator();
			while (it.hasNext()) {
				Step step = (Step) it.next();
				if (! gcDone.contains(step)) {
					this.deleteTrashFromStep(step);
					Collection col = this.getCase().getDependentStepsFor(step);
					gcDone.addAll(col);
				}
			}
		}
		catch (M4Exception m4e)
		{   throw new M4CompilerInterfaceError("M4 interface error in CompilerAccessLogic: " + m4e.getMessage());  }
	}

	/**
     * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#getStatusMessage(long, int)
     */
    public String getStatusMessage(long caseId, int numberOfLines)
	   	throws M4CompilerInterfaceError
    {
    	StringBuffer ret = new StringBuffer("");
	
		// cannot get a status if no case is set yet:
		if (this.casePrintObject == null) {
			return "";
		}
    	final String fileName = this.casePrintObject.getFileName();
		BufferedReader in;
    	try {
    		final File f = new File(fileName);
    		if ( ! f.exists())
    		{   f.createNewFile();  }
			in = new BufferedReader(new FileReader(f));
    	}
    	catch (IOException e) {
    		return "";
    	}
    	
		LinkedList ll = new LinkedList();
		String s;
    	try {
			while ((s = in.readLine()) != null) {
				ll.addLast(s);
				if (ll.size() > numberOfLines) {
					ll.removeFirst();	
				}
			}
			Iterator it = ll.iterator();
			while (it.hasNext()) {
				ret.append((String) it.next());
				ret.append("\n");
			}
    	}
    	catch (IOException e) {
    		// These exceptions are usually harmless, but are propagated to the user:
    		// throw new M4CompilerInterfaceError(e.getMessage());
    	}
		return ret.toString();
    }

	/**
	 * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#getStatusFromLine(long, int)
	 * */
	public String getStatusFromLine(long caseId, int lineNumber)
		throws M4CompilerInterfaceError
	{
		final StringBuffer sb = new StringBuffer(100);
		BufferedReader lnr = null;
		int curentLineNr = 0;
		// cannot get a status if no case is set yet:
		if (this.casePrintObject == null) {
			return "";
		}
		try {
    		final String fileName = this.casePrintObject.getFileName();	
    		//final File f = new File(fileName);
    		//if ( ! f.exists())
    		//{   f.createNewFile();  }
			lnr = new BufferedReader(new FileReader(fileName));
			String line;
			while ((line = lnr.readLine()) != null) {
				if (curentLineNr++ >= lineNumber) {
					sb.append(line + "\n");
				}
				curentLineNr++;
			}
		}
		catch (IOException e) {
			throw new M4CompilerInterfaceError(e.getMessage());
		}
		finally {
			if (lnr != null) {
				try {
					lnr.close();	
				} catch (IOException e) {}
			}	
		}

		return ++curentLineNr + "\n" + sb.toString();
	}
	
	/**
	 * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#changeCompilerOutputVerbosity(long, int)
	 * */
	public void changeCompilerOutputVerbosity(long caseId, Level newVerbosity) throws M4CompilerInterfaceError {
		// compiler output is case-dependent
		this.setCase(caseId);
	    this.casePrintObject.setMinimumVerbosityLevel(newVerbosity);
	}
	
	/**
     * @see edu.udo.cs.miningmart.compiler.CompilerAccessImpl#isReadyForCompilation(long)
     */
	public boolean isReadyForCompilation(long stepId)
	   	throws M4CompilerInterfaceError
	{
		try {
			final Step step = (Step) this.getDb().getM4Object(stepId, Step.class);
			if (step == null) {
				throw new M4CompilerInterfaceError(
					"CompilerAccessLogic.isReadyForCompilation: Step with ID "
					+ stepId + " not found!");
			}
			
			if (step.isCompiled()) {
				return true;
			}
			else {
				edu.udo.cs.miningmart.m4.Case m4Case = step.getTheCase();
				Iterator it = m4Case.getReverseIterator();
				
				// skip all steps up to the one we are investigating
				while (it.hasNext() && ! ((M4Object) step).equals(it.next()));
				
				// check for all "earlier" steps if they are direct
				// predecessors of the target step and if they have
				// been compiled already:
				while (it.hasNext()) {
					Step pid = (Step) it.next();
					if (pid != null && m4Case.containsDependency(pid, step)
						&& pid.isCompiled() == false)
					{ // uncompiled direct predecessor found:
						return false;
					}
				}				
				// No uncompiled predecessor found:
				return true;
			}
		} catch (M4Exception e) {
			throw new M4CompilerInterfaceError(e.getMessage());
		}
	}

	/** 
	 * A call to this method compiles only those steps of a case which have not
	 * already been compiled.
	 * 
	 * Please note that for eager mode it is not checked, whether all steps
	 * compiled earlier were also compiled in eager mode!
	 * 
	 * @param caseId the ID of the case to be compiled
	 * @param lazyMode indicates if the steps should be compiled in lazy mode
	 * */
	public void compileRestOfCase(long caseId, boolean lazyMode)
		throws M4CompilerInterfaceError, M4CompilerWarning 
	{
		try {
			String warningMessage = "";
			this.setCase(caseId);
			Iterator steps = this.getCase().getStepIterator();
			final HashSet garbageCollected = new HashSet();
			while (steps.hasNext()) {
				Step currentStep = (Step) steps.next();
				if (!currentStep.isCompiled() && (this.isReadyForCompilation(currentStep.getId())))
				{
					if (!garbageCollected.contains(currentStep))
					{
						this.deleteTrashBeforeCompilingStep(currentStep);
						Collection col = this.getCase().getDependentStepsFor(currentStep);
						garbageCollected.addAll(col);
					}
					this.printBeforeStepInfo(currentStep, lazyMode);
					try {
						compileStepMultiStepControl(currentStep, lazyMode, false);
					}
					catch (M4CompilerWarning mw) {
						warningMessage += mw.getMessage() + "\n";
					}
					this.printStepExecInfo(currentStep, lazyMode);
				}
			}
			if ( ! warningMessage.equals("")) {
				throw new M4CompilerWarning(warningMessage);
			}
		}
		catch (M4Exception e) {
			throw new M4CompilerInterfaceError(
				"Caught an M4CompilerError trying to compile case "
					+ caseId + "\n"	+ e.getMessage());			
		}
	}
    // ---------------------------------------------------------------------
    
	/**
	 * Gets the instance of class <code>DB</code> representing the database
	 * access to both the M4 and business schema.
	 * 
	 * @return the DB object for the current thread
	 */
	public CompilerDatabaseService getDb() {
		return m4db;
	}

	/** Same as getDb(), but with less specific return type for the implemented interface. */
	public DB getM4db() { return m4db; }


   	/**
	 * Gets the instance of the <code>Case</code>-dependent <code>Print</code>
	 * object, if available. If no such object could be set up, yet, then the
	 * default <code>Print</code> object is returned instead.
	 * 
	 * @return a <code>Print</code> object to be used in the current compiler run
	 */
	public Print getCurrentPrintObject() {
		if (this.casePrintObject != null) {
			return this.casePrintObject;
		}
		else return this.defaultPrintObject;
	}

	/**
	 * Initializes the default <code>Print<code> class of this instance of
	 * <code>CompilerAccessLogic</code>.
	 **/
	private void initDefaultPrintObject() {
		if (this.verbosity != null) {
			this.defaultPrintObject = Print.getGlobalPrintObject();
			this.defaultPrintObject.setMinimumVerbosityLevel(this.verbosity);
		}
		else {
			this.defaultPrintObject = Print.getGlobalPrintObject();
		}
	}


	/**
	 * Initializes the <code>Case</code>-dependent <code>Print<code> class of
	 * this instance of <code>CompilerAccessLogic</code>. In case of errors the
	 * default <code>Print<code> object is used after giving a warning.
	 **/
	private void initCasePrintObject(long caseId) {
		if (this.casePrintObject == null) {
			if (this.verbosity != null) {
				// this.casePrintObject = new Print(caseId, this.verbosity.intValue());
				this.casePrintObject = Print.getGlobalPrintObject();
				this.casePrintObject.setMinimumVerbosityLevel(this.verbosity);
			}
			else {
				//this.casePrintObject = new Print(caseId);
				this.casePrintObject = Print.getGlobalPrintObject();
			}
		}
	}

	/**
	 * Sets an internal flag to remember to stop the current thread next time it
	 * tries to access the database.
	 * */
	void setStopRequest() {
		this.stopThread = true;
	}

	/**
	 * Checked to decide, if an exception should be thrown to stop execution.
	 * @return <code>true</code> iff the Thread should not continue.
	 * */
	public boolean getStopRequest() {
		return this.stopThread;
	}

	/**
	 * This method reads the Case with the given id from the database and
	 * stores it in the private field <i>caseObj</i>. Additionally the
	 * <code>Case</code>-dependent <code>Print</code> object is initialzed.
	 * */
	private void setCase(long caseId) throws M4CompilerInterfaceError {
		this.initCasePrintObject(caseId); // inits only if not existing yet
		try {
			this.caseObj = (Case) this.getM4db().getM4Object(caseId, Case.class);
		}
		catch (M4Exception e) {
			throw new M4CompilerInterfaceError(e.getClass()
				+ " while loading case for caseId " + caseId
				+ "\nin method CompilerAccessLogic.setCase(caseId):\n"
				+ e.getMessage());
		}
	}

	/**
	 * @return the <code>Case</code> associated with this CompilerAccessLogic object. 
	 */	
	public Case getCase() {
		return this.caseObj;
	}

	/** A helper method to get the id of the case embedding the step with the
	 * specified id.
	 * @param stepId the id of the step
	 * @return the id of the case
	 * */
	private long getCaseIdByStepId(long stepId) throws M4CompilerInterfaceError {
		try {
			return this.getDb().readCaseIdForStepId(stepId);
		}
		catch (Exception e) {
			throw new M4CompilerInterfaceError
			(e.getClass() + " while loading caseId for stepId " + stepId
			+ "\nin method CompilerAccessLogic.getCaseIdByStepId(stepId):\n"
			+ e.getMessage());	
		}
	}

	/**
	 * @see miningmart.compiler.utils.CompilerAccessLogic.getPrintObject()
	 */
	public Print getPrintObject() {
		return this.getCurrentPrintObject();
	}
	
	/** A helper method using the <code>doPrint</code> method of the case dependent
	 * <code>Print</code> object. */
	private void doPrint(Level verbosity, String message) {
		this.getCurrentPrintObject().doPrint(verbosity, message);
	}

	/** A helper method using the <code>doPrint</code> method of the case dependent
	 * <code>Print</code> object. */
	private void doPrint(Exception e) {
		this.getCurrentPrintObject().doPrint(Print.ERROR,e.getMessage(),e);
	}
	

	// Inherited but in this context unsupported methods:
	
	public void compileAll(long caseId, boolean lazy, int verbosity) 
	throws M4CompilerInterfaceError, M4CompilerWarning, UserError {
		this.compileAll(caseId, lazy);
	}
	
	public void compileStep(long stepId, boolean lazy, int verbosity)
		throws M4CompilerInterfaceError, M4CompilerWarning, UserError
	{
		this.compileStep(stepId, lazy);
	}

	public void compileStepFrom(long stepId, boolean lazy, int verbosity) throws M4CompilerInterfaceError, M4CompilerWarning, UserError {
		this.compileStepFrom(stepId, lazy);
	}

	public boolean killCompilerThread(long caseId) {
		return false;
	}	
}
/*
 * Historie
 * --------
 *
 * $Log: CompilerAccessLogic.java,v $
 * Revision 1.11  2006/04/11 14:10:09  euler
 * Updated license text.
 *
 * Revision 1.10  2006/04/06 16:31:09  euler
 * Prepended license remark.
 *
 * Revision 1.9  2006/03/30 16:07:16  scholz
 * fixed author tags for release
 *
 * Revision 1.8  2006/03/24 13:14:58  euler
 * Bugfix
 *
 * Revision 1.7  2006/03/23 11:13:44  euler
 * Improved exception handling.
 *
 * Revision 1.6  2006/03/19 21:13:47  scholz
 * UserErrors are propagated now, useful for the GUI
 *
 * Revision 1.5  2006/03/19 17:00:37  scholz
 * refactoring
 *
 * Revision 1.4  2006/02/01 14:58:39  euler
 * bugfix
 *
 * Revision 1.3  2006/01/18 16:58:58  euler
 * Added some basic estimations of statistics.
 * Will need improvements.
 *
 * Revision 1.2  2006/01/12 20:35:18  scholz
 * bugfix statistics
 *
 * Revision 1.1  2006/01/03 09:54:31  hakenjos
 * Initial version!
 *
 */
