Revision 29140
Added by Eri Katsari over 10 years ago
| modules/dnet-openaire-stats/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/drivers/SqoopDriver.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.mapreduce.hbase.statsExport.drivers; |
| 2 | 2 |
|
| 3 | 3 |
import java.io.File; |
| 4 |
import java.io.IOException; |
|
| 4 | 5 |
import java.io.InputStream; |
| 5 | 6 |
import java.util.ArrayList; |
| 6 | 7 |
import java.util.HashMap; |
| 8 |
import java.util.List; |
|
| 7 | 9 |
import java.util.Map.Entry; |
| 8 | 10 |
import java.util.Properties; |
| 9 | 11 |
|
| ... | ... | |
| 17 | 19 |
import org.apache.log4j.Logger; |
| 18 | 20 |
import org.apache.sqoop.tool.SqoopTool; |
| 19 | 21 |
|
| 22 |
import com.cloudera.sqoop.Sqoop; |
|
| 20 | 23 |
import com.cloudera.sqoop.SqoopOptions; |
| 21 | 24 |
import com.cloudera.sqoop.mapreduce.ExportOutputFormat; |
| 25 |
import com.cloudera.sqoop.tool.ExportTool; |
|
| 22 | 26 |
|
| 23 | 27 |
public class SqoopDriver {
|
| 24 | 28 |
private Logger log = Logger.getLogger(this.getClass()); |
| ... | ... | |
| 98 | 102 |
// CommonArgs.addHadoopFlags(args); |
| 99 | 103 |
|
| 100 | 104 |
|
| 101 |
args.add(ExportOutputFormat.RECORDS_PER_STATEMENT_KEY + "=" |
|
| 102 |
+ "1000"); |
|
| 105 |
args.add(ExportOutputFormat.RECORDS_PER_STATEMENT_KEY + "=" + "1000"); |
|
| 103 | 106 |
args.add("-D");
|
| 104 |
args.add(ExportOutputFormat.STATEMENTS_PER_TRANSACTION_KEY + "=" |
|
| 105 |
+ "1000"); |
|
| 107 |
args.add(ExportOutputFormat.STATEMENTS_PER_TRANSACTION_KEY + "=" + "1000"); |
|
| 106 | 108 |
opts.setExtraArgs( args.toArray(new String[0])); |
| 107 | 109 |
|
| 108 | 110 |
conf.set("-Dsqoop.statements.per.transaction", "1000");
|
| ... | ... | |
| 111 | 113 |
|
| 112 | 114 |
tool.appendArgs( args.toArray(new String[0])); |
| 113 | 115 |
// |
| 114 |
tool.run(opts); |
|
| 115 |
|
|
| 116 |
cleanUp(table.getKey()); |
|
| 116 |
// tool.run(opts); |
|
| 117 |
|
|
| 118 |
ExportTool exportTool= new ExportTool(); |
|
| 119 |
|
|
| 120 |
Sqoop sqoop = new Sqoop( (com.cloudera.sqoop.tool.SqoopTool) tool, conf,opts); |
|
| 121 |
Sqoop.runSqoop(sqoop, args.toArray(new String[0])); |
|
| 122 |
cleanUp(table.getKey()); |
|
| 117 | 123 |
} |
| 118 | 124 |
} |
| 119 |
|
|
| 125 |
|
|
| 126 |
|
|
| 120 | 127 |
public static void main(String[] args) {
|
| 121 | 128 |
SqoopDriver sqoopDriver = new SqoopDriver(); |
| 122 | 129 |
|
Also available in: Unified diff