| 11 |
11 |
import javax.annotation.Resource;
|
| 12 |
12 |
import javax.xml.ws.wsaddressing.W3CEndpointReference;
|
| 13 |
13 |
|
|
14 |
import org.antlr.stringtemplate.StringTemplate;
|
| 14 |
15 |
import org.apache.commons.codec.binary.Base64;
|
| 15 |
16 |
import org.apache.commons.lang.StringUtils;
|
| 16 |
17 |
import org.apache.commons.logging.Log;
|
| ... | ... | |
| 53 |
54 |
import eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory;
|
| 54 |
55 |
import eu.dnetlib.functionality.modular.ui.AbstractAjaxController;
|
| 55 |
56 |
import eu.dnetlib.miscutils.datetime.DateUtils;
|
|
57 |
import eu.dnetlib.pace.util.DedupConfig;
|
|
58 |
import eu.dnetlib.pace.util.DedupConfigLoader;
|
| 56 |
59 |
|
| 57 |
60 |
@Controller
|
| 58 |
61 |
public class DedupServiceInternalController extends AbstractAjaxController {
|
| ... | ... | |
| 80 |
83 |
@Value("${dnet.dedup.index.collection}")
|
| 81 |
84 |
private String dedupIndexCollection;
|
| 82 |
85 |
|
|
86 |
@Resource(name = "xqueryDedupWfTemplate")
|
|
87 |
private StringTemplate xqueryWf;
|
|
88 |
|
| 83 |
89 |
private IndexClient indexClient = null;
|
| 84 |
90 |
|
| 85 |
91 |
public class OafResult {
|
| ... | ... | |
| 212 |
218 |
// index update
|
| 213 |
219 |
final CloudSolrServer solrServer = getSolrServer();
|
| 214 |
220 |
final ProtoDocumentMapper mapper = initProtoMapper();
|
|
221 |
final DedupConfig dedupConf = DedupConfigLoader.load(loadWfConf(group.getEntityType().getType(), "01"));
|
| 215 |
222 |
|
| 216 |
223 |
for (final String rootId : group.getRootIds()) {
|
| 217 |
224 |
solrServer.deleteById(rootId);
|
| ... | ... | |
| 222 |
229 |
final List<Oaf> groupDocs = Lists.newArrayList(markDeleted(asOafBuilder(parseBase64(queryIndex(group.getGroup())))));
|
| 223 |
230 |
|
| 224 |
231 |
buffer.addAll(Lists.newArrayList(asIndexDocs(oaf2solr, groupDocs)));
|
| 225 |
|
buffer.add(oaf2solr.apply(new OafEntityMerger().mergeEntities(newRootId(group), groupDocs).build()));
|
|
232 |
buffer.add(oaf2solr.apply(OafEntityMerger.merge(dedupConf, newRootId(group), groupDocs).build()));
|
| 226 |
233 |
|
| 227 |
234 |
final List<Oaf> dissimDocs = Lists.newArrayList(markUnDeleted(asOafBuilder(parseBase64(queryIndex(group.getDissimilar().keySet())))));
|
| 228 |
235 |
buffer.addAll(Lists.newArrayList(asIndexDocs(oaf2solr, dissimDocs)));
|
| ... | ... | |
| 495 |
502 |
|
| 496 |
503 |
private String getIndexSolrUrlZk() {
|
| 497 |
504 |
try {
|
| 498 |
|
return getServiceConfigValue("for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//PROTOCOL[./@name='solr']/@address/string()");
|
|
505 |
return getResourceProfileByQuery("for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//PROTOCOL[./@name='solr']/@address/string()");
|
| 499 |
506 |
} catch (final ISLookUpException e) {
|
| 500 |
507 |
throw new IllegalStateException("unable to read solr ZK url from service profile", e);
|
| 501 |
508 |
}
|
| 502 |
509 |
}
|
| 503 |
510 |
|
| 504 |
|
private String getServiceConfigValue(final String xquery) throws ISLookUpException {
|
|
511 |
private String getResourceProfileByQuery(final String xquery) throws ISLookUpException {
|
| 505 |
512 |
log.debug("quering for service property: " + xquery);
|
| 506 |
513 |
final String res = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
|
| 507 |
514 |
if (StringUtils.isBlank(res)) throw new IllegalStateException("unable to find unique service property, xquery: " + xquery);
|
| 508 |
515 |
return res;
|
| 509 |
516 |
}
|
| 510 |
517 |
|
|
518 |
private String getXquery(final String entityType, final String code, final StringTemplate template) {
|
|
519 |
final StringTemplate xquery = new StringTemplate(template.getTemplate());
|
|
520 |
xquery.setAttribute("entityType", entityType);
|
|
521 |
xquery.setAttribute("configCode", code);
|
|
522 |
return xquery.toString();
|
|
523 |
}
|
|
524 |
|
|
525 |
private String loadWfConf(final String entityType, final String conf) throws ISLookUpException {
|
|
526 |
return getResourceProfileByQuery(getXquery(entityType, conf, xqueryWf));
|
|
527 |
}
|
|
528 |
|
| 511 |
529 |
}
|
Added by Claudio Atzori over 9 years ago
using dedupConf driven OafEntityMerger