com.datasalt.pangool.utils.test
Class AbstractHadoopTestLibrary

java.lang.Object
  extended by com.datasalt.pangool.utils.test.AbstractBaseTest
      extended by com.datasalt.pangool.utils.test.AbstractHadoopTestLibrary

public abstract class AbstractHadoopTestLibrary
extends AbstractBaseTest

Niceties and utilities for making Hadoop unit tests less painfully.


Nested Class Summary
static class AbstractHadoopTestLibrary.PrintVisitor
           
static class AbstractHadoopTestLibrary.TupleVisitor
           
 
Field Summary
protected  org.apache.hadoop.fs.FileSystem fS
           
protected  Map<String,Object> inputs
           
protected  Map<String,List<Pair<Object,Object>>> outputs
           
 
Constructor Summary
AbstractHadoopTestLibrary()
           
 
Method Summary
 void assertRun(org.apache.hadoop.mapreduce.Job job)
           
 void cleanUp()
           
 void dumpOutput(String output)
          Dumps to string the given output
 List<Pair<Object,Object>> ensureOutput(String output)
           
protected  String firstMapOutput(String path)
           
protected  String firstMapperMultiOutput(String path, String multiOutputName)
           
protected  String firstReducerMultiOutput(String path, String multiOutputName)
           
protected  String firstReducerOutput(String path)
           
 void initHadoop()
           
static void readTuples(org.apache.hadoop.fs.Path file, org.apache.hadoop.conf.Configuration conf, AbstractHadoopTestLibrary.TupleVisitor iterator)
           
protected  void trash(String... folders)
           
 AbstractHadoopTestLibrary withInput(String input, Object key)
           
 AbstractHadoopTestLibrary withInput(String input, Object key, Object value)
           
 void withOutput(String output, Object key)
           
 void withOutput(String output, Object key, Object value)
           
 AbstractHadoopTestLibrary withTupleInput(String input, ITuple tuple)
           
 void withTupleOutput(String output, ITuple expectedTuple)
           
 org.apache.hadoop.io.Writable writable(Object obj)
           
 
Methods inherited from class com.datasalt.pangool.utils.test.AbstractBaseTest
createConf, createNewConfiguration, getConf, prepare
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

fS

protected org.apache.hadoop.fs.FileSystem fS

outputs

protected Map<String,List<Pair<Object,Object>>> outputs

inputs

protected Map<String,Object> inputs
Constructor Detail

AbstractHadoopTestLibrary

public AbstractHadoopTestLibrary()
Method Detail

initHadoop

public void initHadoop()
                throws IOException
Throws:
IOException

writable

public org.apache.hadoop.io.Writable writable(Object obj)

assertRun

public void assertRun(org.apache.hadoop.mapreduce.Job job)
               throws IOException,
                      InterruptedException,
                      ClassNotFoundException
Throws:
IOException
InterruptedException
ClassNotFoundException

cleanUp

public void cleanUp()
             throws IOException
Throws:
IOException

trash

protected void trash(String... folders)
              throws IOException
Throws:
IOException

firstReducerOutput

protected String firstReducerOutput(String path)

firstMapOutput

protected String firstMapOutput(String path)

firstReducerMultiOutput

protected String firstReducerMultiOutput(String path,
                                         String multiOutputName)

firstMapperMultiOutput

protected String firstMapperMultiOutput(String path,
                                        String multiOutputName)

withInput

public AbstractHadoopTestLibrary withInput(String input,
                                           Object key,
                                           Object value)
                                    throws IOException
Throws:
IOException

withInput

public AbstractHadoopTestLibrary withInput(String input,
                                           Object key)
                                    throws IOException
Throws:
IOException

withTupleInput

public AbstractHadoopTestLibrary withTupleInput(String input,
                                                ITuple tuple)
                                         throws IOException
Throws:
IOException

withOutput

public void withOutput(String output,
                       Object key)
                throws IOException,
                       ClassNotFoundException,
                       InstantiationException,
                       IllegalAccessException
Throws:
IOException
ClassNotFoundException
InstantiationException
IllegalAccessException

readTuples

public static void readTuples(org.apache.hadoop.fs.Path file,
                              org.apache.hadoop.conf.Configuration conf,
                              AbstractHadoopTestLibrary.TupleVisitor iterator)
                       throws IOException,
                              InterruptedException
Throws:
IOException
InterruptedException

withTupleOutput

public void withTupleOutput(String output,
                            ITuple expectedTuple)
                     throws IOException,
                            InterruptedException
Throws:
IOException
InterruptedException

withOutput

public void withOutput(String output,
                       Object key,
                       Object value)
                throws IOException
Throws:
IOException

ensureOutput

public List<Pair<Object,Object>> ensureOutput(String output)
                                       throws IOException
Throws:
IOException

dumpOutput

public void dumpOutput(String output)
                throws IOException
Dumps to string the given output

Throws:
IOException


Copyright © –2014 Datasalt Systems S.L.. All rights reserved.