---

Classifier.java

gehe zur Dokumentation dieser Datei

/*
 * $Source: /shared/cvsroot/diplom/app/src/java/de/picana/classifier/Classifier.java,v $
 * $Author: mstolpe $
 * $Date: 2003/04/22 09:51:27 $
 * $Revision: 1.8 $
 * $Release$ 
 *
 * Created on 01. November 2002, 18:00
 *
 * Copyright 2002 by Marco Stolpe
 */

package de.picana.classifier;

import de.picana.clusterer.MLVector;
import de.picana.control.*;
import de.picana.logging.*;
import de.picana.math.*;

import weka.core.*;
import java.io.*;
import java.util.*;
import java.text.*;


public abstract class Classifier extends Task {
 
    protected String infile;
    protected String modelfile;
    protected String centroidfile;
    protected String statfile;
    protected String outfile;
    protected double param_a;
   
    protected int num_clusters;
    
    private int num_attributes;
    private Instances training_set;
    private Instances result_set;
    private Instances[] clusters;
    private double[][] means;

    protected DecimalFormat decf;
    
    
    public Classifier() {
    }

    public void init(ParameterSet params, Logger logger) {
        
        super.init(params, logger);
        
        decf = (DecimalFormat)DecimalFormat.getInstance(Locale.ENGLISH);
        decf.applyPattern("####0.000000");
        
        infile = (String)params.getParameter("in");
        modelfile = (String)params.getParameter("model");
        outfile = (String)params.getParameter("out");
        centroidfile = (String)params.getParameter("centroids");
        statfile = (String)params.getParameter("statistics");
        
        /*
        param_a = 0.0;
        try {
            param_a = Double.parseDouble((String)params.getParameter("a"));
        } catch (NumberFormatException nfe) {}
         */
    }
    
    public void start() throws TaskException {
    
        try {
            logger.info(LOGSRC, "Started.");
            
            logger.info(LOGSRC, "Read infile '" + infile + "' ...");
            File input = new File(infile);
            FileInputStream fis = new FileInputStream(input);
            InputStreamReader reader = new InputStreamReader(fis);
            training_set = new Instances(reader); 
            num_attributes = training_set.numAttributes();
            logger.info(LOGSRC, "reading '" + infile + "' done.");
            
            logger.info(LOGSRC, "Read modelfile '" + modelfile + "' ...");
            loadModel(modelfile);
            logger.info(LOGSRC, "reading '" + modelfile + "' done.");
            
            logger.info(LOGSRC, "Classify instances ...");
                        
            clusters = new Instances[num_clusters];
            means = new double[num_clusters][num_attributes];
            
            for (int i=0; i < training_set.numInstances(); i++) {
            
                Instance ex = training_set.instance(i);
                int cl = classify(ex);

                if (clusters[cl] == null)
                    clusters[cl] = new Instances(training_set, 0);
                
                clusters[cl].add(ex);
            }
            
            logger.info(LOGSRC, "classification done.");
            
            
            logger.info(LOGSRC, "Calculate meta data ...");
        
            MLVector vec;
            Integer freq;
            Instance inst;
            int dim;
        
            Map freq_htable = new HashMap();
        
            // build frequency hash table
        
            for (int i=0; i < training_set.numInstances(); i++) {
                inst = training_set.instance(i);
                dim = inst.numAttributes();
            
                // convert instance into vector
                vec = new MLVector(dim);
                for (int j=0; j < dim; j++)
                    vec.value[j] = inst.value(j);   
            
                freq = (Integer)freq_htable.get(vec);
            
                freq_htable.put(vec, (freq==null) ? ONE :
                    new Integer(freq.intValue() + 1));
            }
                        
            double[] min_dists = new double[num_clusters];
            int[] min_cls = new int[num_clusters];
            double[] stats = new double[2];
            
            double empvar = Stats.getEmpVar(training_set) /
                            (training_set.numInstances()-1);
            double sst = Stats.getSST(training_set);
            double ssb = Stats.getSSB(training_set, clusters);
            double ssw = Stats.getSSW(clusters);
            double mmd = Distance.getMMD(clusters, min_dists, min_cls);
            double gmmd_a1 = Distance.getGMMD(clusters, 0.1, min_dists, min_cls, stats);
            double gmmd_a5 = Distance.getGMMD(clusters, 0.5, min_dists, min_cls, stats);
            double gmmd_a9 = Distance.getGMMD(clusters, 0.9, min_dists, min_cls, stats);
            
            logger.info(LOGSRC, "Instances:  " + training_set.numInstances());
            logger.info(LOGSRC, "UniqueInst: " + freq_htable.size());
            logger.info(LOGSRC, "Clusters:   " + num_clusters);
            logger.info(LOGSRC, "EmpVar:     " + decf.format(empvar));
            logger.info(LOGSRC, "SST:        " + decf.format(sst));
            logger.info(LOGSRC, "SSB:        " + decf.format(ssb));
            logger.info(LOGSRC, "SSW:        " + decf.format(ssw));
            logger.info(LOGSRC, "SSB + SSW:  " + decf.format(ssb + ssw));
            logger.info(LOGSRC, "MMD:        " + decf.format(mmd));
            //logger.info(LOGSRC, "GMMD_di:    " + decf.format(stats[0]));
            //logger.info(LOGSRC, "GINI:       " + decf.format(stats[1]));
            logger.info(LOGSRC, "GMMD_a1:    " + decf.format(gmmd_a1));
            logger.info(LOGSRC, "GMMD_a5:    " + decf.format(gmmd_a5));
            logger.info(LOGSRC, "GMMD_a9:    " + decf.format(gmmd_a9));
            logger.info(LOGSRC, "calculations for meta data done.");
                        
            if (statfile != null) {
                
                logger.info(LOGSRC, "Write statistics.");
                
                PrintWriter statw = new PrintWriter(
                    new FileOutputStream(new File(statfile), true));
            
                statw.println("stat_Instances:" + training_set.numInstances());
                statw.println("stat_UniqueInst:" + freq_htable.size());
                statw.println("stat_Clusters:" + num_clusters);
                statw.println("stat_EMPVAR: " + empvar);
                statw.println("stat_SST:" + sst);
                statw.println("stat_SSB:" + ssb);
                statw.println("stat_SSW:" + ssw);
                statw.println("stat_MMD:" + mmd);
                statw.println("stat_GMMD_a1:" + gmmd_a1);
                statw.println("stat_GMMD_a5:" + gmmd_a5);
                statw.println("stat_GMMD_a9:" + gmmd_a9);
            
                statw.close();
            }
            
            logger.info(LOGSRC, "Calculate cluster means ...");
            for (int cl=0; cl < num_clusters; cl++) {
                
                if (clusters[cl] != null) {
                    
                    String output = "cluster[" + cl + "] = (";
                    
                    for (int att=0; att < num_attributes; att++) { 
                    
                        means[cl][att] = clusters[cl].meanOrMode(att);
                        output += decf.format(means[cl][att]);
                        if (att != num_attributes-1)
                            output += ", ";
                    }
                    
                    output += ") -> " + clusters[cl].numInstances();
                
                    logger.info(LOGSRC, output);
                }
            }
            logger.info(LOGSRC, "calculations for cluster means done.");

            if (centroidfile != null) {
                
                logger.info(LOGSRC, "Write centroids.");    
            
                PrintWriter centroidsw = new PrintWriter(
                    new FileOutputStream(new File(centroidfile)));
                
                centroidsw.println("Clusters:" + num_clusters);
                for (int att=0; att < num_attributes; att++) {
                    centroidsw.print(training_set.attribute(att).name());
                    if (att != num_attributes-1)
                        centroidsw.print(",");
                }
                centroidsw.println(":instances");
                
                for (int cl=0; cl < num_clusters; cl++) {
                    
                    if (clusters[cl] != null) {
                        for (int att=0; att < num_attributes; att++) {
                            centroidsw.print(means[cl][att]);
                            if (att != num_attributes-1)
                                centroidsw.print(",");
                        }
                        centroidsw.println(":" + clusters[cl].numInstances());
                    } else {
                        for (int att=0; att < num_attributes; att++) {
                            centroidsw.print(0.0);
                            if (att != num_attributes-1)
                                centroidsw.print(",");
                        }
                        centroidsw.println(":0");
                    }
                }
                
                centroidsw.close();
            }
            
            logger.info(LOGSRC, "Classify instances and replace them by cluster means ...");
            
            result_set = new Instances(training_set, 0);
            
            for (int i=0; i < training_set.numInstances(); i++) {
            
                Instance ex = training_set.instance(i);
                Instance res = new Instance(ex);
                int cl = classify(ex);

                for (int att=0; att < num_attributes; att++)                
                    res.setValue(att, means[cl][att]);

                result_set.add(res);
            }
            
            for (int cl=0; cl < num_clusters; cl++) {
                if (clusters[cl] != null)
                    clusters[cl] = null;
            }
            clusters = null;
            training_set = null;
            
            logger.info(LOGSRC, "all instances replaced.");
            
            logger.info(LOGSRC, "Write classified instances to outfile '" + outfile + "'");
            
            File output = new File(outfile);
            PrintWriter pw = new PrintWriter(new FileOutputStream(output));
            
            pw.println("% ARFF file");
            pw.println("%");
            pw.println("@relation " + result_set.relationName());
            pw.println();
            for (int att=0; att < num_attributes; att++)
                pw.println("@attribute " + result_set.attribute(att).name() + " numeric");
            pw.println();
            pw.println("@data");
            pw.println("%");
            pw.println("% x instances");
            pw.println("%");
            pw.flush();
            
            for (int i=0; i < result_set.numInstances(); i++) {
            
                Instance res = result_set.instance(i);
                String line = "";
                for (int att=0; att < num_attributes; att++)
                    line += res.value(att) + ",";
                line = line.substring(0, line.length()-1);                
                pw.println(line);
                pw.flush();
                //System.out.println(i);
                System.out.flush();
            }
            
            pw.close();
            
            logger.info(LOGSRC, "classified instances written to '" + outfile + "'");
            
            logger.info(LOGSRC, "Stopped.");
        
        } catch (Exception e) {
            logger.error(LOGSRC, e.toString());
            throw new TaskException(e.toString());
        }    
    }
    
    public void stop() {
    }
    
    public void pause() {
    }
    
    public void resume() {
    }
    
    protected abstract void loadModel(String filename) throws TaskException;
   
    protected abstract int classify(Instance i);
}

---