Revision 41140
Added by Giorgos Alexiou almost 9 years ago
| modules/dnet-openaire-lodimport/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/lodImport/utils/RDFizer.java | ||
|---|---|---|
| 140 | 140 |
|
| 141 | 141 |
} |
| 142 | 142 |
|
| 143 |
insertQuery+="."; |
|
| 143 |
insertQuery+=". ";
|
|
| 144 | 144 |
|
| 145 | 145 |
return insertQuery; |
| 146 | 146 |
|
| ... | ... | |
| 173 | 173 |
exists = stmt.execute(askQuery); |
| 174 | 174 |
// if(rs.next()) exists=true; |
| 175 | 175 |
stmt.close(); |
| 176 |
conn.commit(); |
|
| 176 |
// conn.commit();
|
|
| 177 | 177 |
conn.close(); |
| 178 | 178 |
} catch (Exception e) {
|
| 179 |
log.error("Virtuoso ask Query failed. Checkpoint was "+ askQuery +"\n" + e.toString(), e);
|
|
| 179 |
log.error("Virtuoso ask Query failed. Query was "+ askQuery +"\n" + e.toString(), e);
|
|
| 180 | 180 |
} |
| 181 | 181 |
return exists; |
| 182 | 182 |
} |
| ... | ... | |
| 187 | 187 |
try {
|
| 188 | 188 |
stmt = conn.createStatement(); |
| 189 | 189 |
stmt.execute(deleteQueryString); |
| 190 |
conn.commit(); |
|
| 190 | 191 |
stmt.close(); |
| 191 |
conn.commit(); |
|
| 192 | 192 |
conn.close(); |
| 193 | 193 |
} catch (Exception e) {
|
| 194 | 194 |
log.error("Virtuoso ask Query failed. Checkpoint was "+ deleteQueryString +"\n" + e.toString(), e);
|
| ... | ... | |
| 201 | 201 |
try {
|
| 202 | 202 |
stmt = conn.createStatement(); |
| 203 | 203 |
stmt.execute(clearQuery); |
| 204 |
conn.commit(); |
|
| 204 | 205 |
stmt.close(); |
| 205 |
conn.commit(); |
|
| 206 |
conn.close(); |
|
| 206 |
// conn.commit();
|
|
| 207 |
// conn.close();
|
|
| 207 | 208 |
} catch (Exception e) {
|
| 208 | 209 |
log.error("Virtuoso FAILED TO CLEAR graph "+ clearQuery +"\n" + e.toString(), e);
|
| 209 | 210 |
} |
| 210 | 211 |
} |
| 211 | 212 |
|
| 212 |
public static void setCheckpoint(Connection conn, String checkpointValue){
|
|
| 213 |
public static void setCheckpoint(Connection conn, int checkpointValue){
|
|
| 213 | 214 |
Statement stmt; |
| 214 | 215 |
try {
|
| 215 | 216 |
stmt = conn.createStatement(); |
| 216 | 217 |
stmt.execute("checkpoint_interval("+checkpointValue+");");
|
| 218 |
conn.commit(); |
|
| 219 |
// stmt.close(); |
|
| 220 |
// stmt = conn.createStatement(); |
|
| 221 |
stmt.execute("checkpoint;");
|
|
| 222 |
conn.commit(); |
|
| 217 | 223 |
stmt.close(); |
| 218 |
conn.commit();
|
|
| 219 |
conn.close(); |
|
| 224 |
//
|
|
| 225 |
// conn.close();
|
|
| 220 | 226 |
} catch (Exception e) {
|
| 221 | 227 |
log.error("Virtuoso set checkpoint failed. Checkpoint was "+ checkpointValue +"\n" + e.toString(), e);
|
| 222 | 228 |
} |
| modules/dnet-openaire-lodimport/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/lodImport/LodImportReducer.java | ||
|---|---|---|
| 2 | 2 |
|
| 3 | 3 |
import java.io.IOException; |
| 4 | 4 |
import java.sql.Connection; |
| 5 |
import java.sql.PreparedStatement; |
|
| 6 |
import java.sql.SQLException; |
|
| 5 | 7 |
import java.sql.Statement; |
| 6 | 8 |
import java.util.ArrayList; |
| 7 | 9 |
import java.util.Arrays; |
| ... | ... | |
| 57 | 59 |
DB db = new DB(); |
| 58 | 60 |
ds = db.getDatasource(hadoopConf.get("lod.conLine"), hadoopConf.get("lod.username"), hadoopConf.get("lod.password"), hadoopConf.get("lod.minCpart"), hadoopConf.get("lod.maxCpart"), hadoopConf.get("lod.part"));
|
| 59 | 61 |
conn = ds.getConnection(); |
| 62 |
conn.setAutoCommit(false); |
|
| 60 | 63 |
} catch (Exception e) {
|
| 61 | 64 |
log.error(e.toString(), e); |
| 62 | 65 |
|
| ... | ... | |
| 95 | 98 |
// System.out.println("so FAR: "+buildQuery);
|
| 96 | 99 |
counter++; |
| 97 | 100 |
if (counter > entitiesPerQuery) {
|
| 98 |
String insertQuery = "SPARQL INSERT INTO <" + defaultGraph + "> { " + buildQuery + "}";
|
|
| 99 |
Statement stmt; |
|
| 100 |
// Connection conn; |
|
| 101 |
// String insertQuery = "SPARQL INSERT INTO <" + defaultGraph + "> { " + buildQuery + "}";
|
|
| 102 |
// Statement stmt;
|
|
| 103 |
//// Connection conn;
|
|
| 101 | 104 |
try {
|
| 102 |
// conn = ds.getConnection(); |
|
| 103 |
stmt = conn.createStatement(); |
|
| 104 |
stmt.execute(insertQuery); |
|
| 105 |
stmt.close(); |
|
| 105 |
|
|
| 106 |
|
|
| 107 |
PreparedStatement ps = conn.prepareStatement("DB.DBA.TTLP_MT (?, ?, ?, ?, ?, ?,?)");
|
|
| 108 |
ps.setString(1, buildQuery); |
|
| 109 |
ps.setString(2, ""); |
|
| 110 |
ps.setString(3, defaultGraph); |
|
| 111 |
final int NQUAD_LEVEL = 64; |
|
| 112 |
ps.setInt(4, NQUAD_LEVEL); |
|
| 113 |
ps.setInt(5, 1); |
|
| 114 |
ps.setInt(6, 10); |
|
| 115 |
ps.setInt(7, 0); |
|
| 116 |
ps.execute(); |
|
| 117 |
ps.close(); |
|
| 118 |
|
|
| 119 |
conn.commit(); |
|
| 120 |
// |
|
| 106 | 121 |
// conn.close(); |
| 107 | 122 |
buildQuery = ""; |
| 108 | 123 |
commitCounter+=counter; |
| 109 | 124 |
counter = 0; |
| 110 |
if(commitCounter>100){
|
|
| 111 |
commitCounter=0; |
|
| 112 |
System.err.println("commit Started "+commitCounter);
|
|
| 113 |
conn.commit(); |
|
| 125 |
|
|
| 126 |
// if(commitCounter>100){
|
|
| 127 |
// commitCounter=0; |
|
| 128 |
//// conn.commit(); |
|
| 129 |
// conn.close(); |
|
| 130 |
// conn=ds.getConnection(); |
|
| 131 |
// } |
|
| 132 |
// |
|
| 133 |
} catch (Exception e) {
|
|
| 134 |
// log.error("Virtuoso write failed at query" + insertQuery + " \n and with error " + e.toString(), e);
|
|
| 135 |
log.error("Virtuoso write failed at query with error " + e.toString(), e);
|
|
| 136 |
try {
|
|
| 114 | 137 |
conn.close(); |
| 115 |
conn=ds.getConnection(); |
|
| 116 |
} |
|
| 138 |
conn=ds.getConnection(); |
|
| 139 |
conn.rollback(); |
|
| 140 |
} catch (Exception e1) {
|
|
| 141 |
log.error("Getting another Connection failed:"+ e1.toString(), e1);
|
|
| 142 |
} |
|
| 117 | 143 |
|
| 118 |
} catch (Exception e) {
|
|
| 119 |
log.error("Virtuoso write failed at query" + insertQuery + " \n and with error " + e.toString(), e);
|
|
| 120 | 144 |
} |
| 121 | 145 |
|
| 122 | 146 |
|
| ... | ... | |
| 140 | 164 |
|
| 141 | 165 |
@Override |
| 142 | 166 |
protected void cleanup(Context context) throws IOException, InterruptedException {
|
| 143 |
String insertQuery = "SPARQL INSERT INTO <" + defaultGraph + "> { " + buildQuery + "}";
|
|
| 144 |
System.out.println("CLEAN-> NEW INSERT QUERY: " + insertQuery);
|
|
| 145 |
Statement stmt; |
|
| 146 |
// Connection conn; |
|
| 167 |
// String insertQuery = "SPARQL INSERT INTO <" + defaultGraph + "> { " + buildQuery + "}";
|
|
| 168 |
// System.out.println("CLEAN-> NEW INSERT QUERY: " + insertQuery);
|
|
| 169 |
// Statement stmt;
|
|
| 170 |
//// Connection conn;
|
|
| 147 | 171 |
try {
|
| 148 |
// conn.close(); |
|
| 149 |
// conn = ds.getConnection(); |
|
| 150 |
stmt = conn.createStatement(); |
|
| 151 |
stmt.execute(insertQuery); |
|
| 152 |
stmt.close(); |
|
| 153 |
// Query q = QueryFactory.create(insertQuery); |
|
| 154 |
// VirtuosoUpdateRequest q2ur = VirtuosoUpdateFactory.create(q.serialize(), graph); |
|
| 155 |
// VirtuosoUpdateRequest q2ur = VirtuosoUpdateFactory.create(insertQuery, graph); |
|
| 156 |
// q2ur.exec(); |
|
| 172 |
PreparedStatement ps = conn.prepareStatement("DB.DBA.TTLP_MT (?, ?, ?, ?, ?, ?,?)");
|
|
| 173 |
ps.setString(1, buildQuery); |
|
| 174 |
ps.setString(2, ""); |
|
| 175 |
ps.setString(3, defaultGraph); |
|
| 176 |
final int NQUAD_LEVEL = 64; |
|
| 177 |
ps.setInt(4, NQUAD_LEVEL); |
|
| 178 |
ps.setInt(5, 1); |
|
| 179 |
ps.setInt(6, 50); |
|
| 180 |
ps.setInt(7, 0); |
|
| 181 |
ps.execute(); |
|
| 182 |
ps.close(); |
|
| 183 |
|
|
| 184 |
conn.commit(); |
|
| 157 | 185 |
} catch (Exception e) {
|
| 158 |
log.error("Virtuoso write failed at query" + insertQuery + " \n and with error " + e.toString(), e);
|
|
| 186 |
log.error("Virtuoso write failed at query" + buildQuery + " \n and with error " + e.toString(), e);
|
|
| 159 | 187 |
// throw new InterruptedException("Virtuoso write failed at query" + insertQuery + " \n and with error " + e.toString());
|
| 160 | 188 |
} finally {
|
| 161 | 189 |
log.info("Cleaning up reducer...\nClosing Graph and Datasource");
|
Also available in: Unified diff
DB changes