Revision 37440
Added by Claudio Atzori over 9 years ago
| modules/dnet-deduplication/trunk/src/main/java/eu/dnetlib/functionality/modular/ui/dedup/DedupServiceInternalController.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.functionality.modular.ui.dedup; |
| 2 | 2 |
|
| 3 |
import java.io.IOException; |
|
| 3 | 4 |
import java.util.Collections; |
| 4 | 5 |
import java.util.HashMap; |
| 5 | 6 |
import java.util.List; |
| ... | ... | |
| 28 | 29 |
|
| 29 | 30 |
import com.google.common.base.Function; |
| 30 | 31 |
import com.google.common.base.Joiner; |
| 32 |
import com.google.common.base.Predicate; |
|
| 31 | 33 |
import com.google.common.base.Splitter; |
| 32 | 34 |
import com.google.common.collect.Iterables; |
| 33 | 35 |
import com.google.common.collect.Lists; |
| 34 | 36 |
import com.google.common.collect.Maps; |
| 37 |
import com.google.common.collect.Sets; |
|
| 35 | 38 |
|
| 36 | 39 |
import eu.dnetlib.data.mapreduce.util.OafDecoder; |
| 37 | 40 |
import eu.dnetlib.data.mapreduce.util.OafEntityDecoder; |
| ... | ... | |
| 182 | 185 |
@RequestMapping(value = "/ui/dedup/commit.do") |
| 183 | 186 |
public boolean addSimRels(@RequestBody(required = true) final SimilarityGroup group) throws Exception {
|
| 184 | 187 |
try {
|
| 188 |
if (StringUtils.isBlank(group.getActionSet())) throw new IllegalArgumentException("missing actionset");
|
|
| 185 | 189 |
final DatabaseService dbService = serviceLocator.getService(DatabaseService.class); |
| 186 | 190 |
final String version = InputDocumentFactory.getParsedDateField(DateUtils.now_ISO8601()); |
| 187 | 191 |
|
| ... | ... | |
| 201 | 205 |
final CloudSolrServer solrServer = getSolrServer(); |
| 202 | 206 |
final ProtoDocumentMapper mapper = initProtoMapper(); |
| 203 | 207 |
|
| 204 |
final Map<String, String> config = Maps.newHashMap(); |
|
| 205 |
config.put("entityType", group.getEntityType().getType());
|
|
| 206 |
config.put("configurationId", group.getActionSet());
|
|
| 207 |
final DedupConfig dedupConf = DedupConfig.loadDefault(config); |
|
| 208 |
|
|
| 209 |
for (final String rootId : group.getRootIds()) {
|
|
| 210 |
solrServer.deleteById(rootId); |
|
| 211 |
} |
|
| 212 | 208 |
final Function<Oaf, SolrInputDocument> oaf2solr = oaf2solr(version, group.getActionSet(), mapper); |
| 213 | 209 |
final List<SolrInputDocument> buffer = Lists.newLinkedList(); |
| 214 | 210 |
|
| 215 | 211 |
final List<Oaf> groupDocs = Lists.newArrayList(markDeleted(asOafBuilder(parseBase64(queryIndex(group.getGroup(), group.getActionSet()))))); |
| 216 | 212 |
|
| 217 | 213 |
buffer.addAll(Lists.newArrayList(asIndexDocs(oaf2solr, groupDocs))); |
| 214 |
final DedupConfig dedupConf = getDedupConfig(group); |
|
| 218 | 215 |
final SolrInputDocument newRoot = oaf2solr.apply(OafEntityMerger.merge(dedupConf, newRootId(group), groupDocs).build()); |
| 219 |
newRoot.setField("actionset", dedupConf.getWf().getConfigurationId());
|
|
| 216 |
// newRoot.setField("actionset", dedupConf.getWf().getConfigurationId());
|
|
| 220 | 217 |
buffer.add(newRoot); |
| 221 | 218 |
|
| 219 |
final Set<String> dissimIds = getUniqueDissimilarIds(group.getDissimilar()); |
|
| 222 | 220 |
final List<Oaf> dissimDocs = |
| 223 |
Lists.newArrayList(markUnDeleted(asOafBuilder(parseBase64(queryIndex(group.getDissimilar().keySet(), group.getActionSet())))));
|
|
| 221 |
Lists.newArrayList(markUnDeleted(asOafBuilder(parseBase64(queryIndex(dissimIds, group.getActionSet())))));
|
|
| 224 | 222 |
buffer.addAll(Lists.newArrayList(asIndexDocs(oaf2solr, dissimDocs))); |
| 225 | 223 |
|
| 226 | 224 |
log.debug(String.format("adding %d documents to index %s", buffer.size(), dedupIndexCollection));
|
| ... | ... | |
| 228 | 226 |
final int addStatus = solrServer.add(buffer).getStatus(); |
| 229 | 227 |
log.debug("solr add status: " + addStatus);
|
| 230 | 228 |
|
| 229 |
log.debug(String.format("deleting %d documents from index %s", group.getRootIds().size(), dedupIndexCollection));
|
|
| 230 |
for (final String rootId : Iterables.filter(group.getRootIds(), new Predicate<String>() {
|
|
| 231 |
|
|
| 232 |
final String newRootId = (String) newRoot.getFieldValue("objidentifier");
|
|
| 233 |
|
|
| 234 |
@Override |
|
| 235 |
public boolean apply(final String rootId) {
|
|
| 236 |
|
|
| 237 |
return !rootId.equals(newRootId); |
|
| 238 |
} |
|
| 239 |
})) {
|
|
| 240 |
solrServer.deleteById(mapper.getRecordId(rootId, group.getActionSet())); |
|
| 241 |
} |
|
| 242 |
|
|
| 231 | 243 |
final int commitStatus = solrServer.commit().getStatus(); |
| 232 | 244 |
log.debug("solr commit status: " + commitStatus);
|
| 233 | 245 |
|
| ... | ... | |
| 238 | 250 |
} |
| 239 | 251 |
} |
| 240 | 252 |
|
| 253 |
private Set<String> getUniqueDissimilarIds(final Map<String, Set<String>> map) {
|
|
| 254 |
final Set<String> res = Sets.newHashSet(); |
|
| 255 |
for (final Set<String> ids : map.values()) {
|
|
| 256 |
res.addAll(ids); |
|
| 257 |
} |
|
| 258 |
|
|
| 259 |
return res; |
|
| 260 |
} |
|
| 261 |
|
|
| 262 |
private DedupConfig getDedupConfig(final SimilarityGroup group) throws IOException {
|
|
| 263 |
final Map<String, String> config = Maps.newHashMap(); |
|
| 264 |
config.put("entityType", group.getEntityType().getType());
|
|
| 265 |
config.put("configurationId", group.getActionSet());
|
|
| 266 |
final DedupConfig dedupConf = DedupConfig.loadDefault(config); |
|
| 267 |
return dedupConf; |
|
| 268 |
} |
|
| 269 |
|
|
| 241 | 270 |
// helpers |
| 242 | 271 |
|
| 243 | 272 |
private IndexClient getIndexClient() throws IndexClientException, ISLookUpDocumentNotFoundException, ISLookUpException {
|
| ... | ... | |
| 304 | 333 |
} |
| 305 | 334 |
|
| 306 | 335 |
private String newRootId(final SimilarityGroup group) {
|
| 307 |
if (group.getRootIds().isEmpty()) return "dedup_wf_001::" + Collections.min(group.getGroup()).replaceFirst("^.*::", "");
|
|
| 308 |
else return Collections.min(group.getRootIds()); |
|
| 336 |
return "dedup_wf_001::" + Collections.min(group.getGroup()).replaceFirst("^.*::", "");
|
|
| 337 |
// else return Collections.min(group.getRootIds());
|
|
| 309 | 338 |
} |
| 310 | 339 |
|
| 311 |
private Iterable<SolrInputDocument> asIndexDocs(final Function<Oaf, SolrInputDocument> mapper, final Iterable<Oaf> a) {
|
|
| 312 |
return Iterables.transform(a, mapper);
|
|
| 340 |
private Iterable<SolrInputDocument> asIndexDocs(final Function<Oaf, SolrInputDocument> mapper, final Iterable<Oaf> protos) {
|
|
| 341 |
return Iterables.transform(protos, mapper);
|
|
| 313 | 342 |
} |
| 314 | 343 |
|
| 315 | 344 |
private Function<Oaf, SolrInputDocument> oaf2solr(final String version, final String actionSetId, final ProtoDocumentMapper mapper) {
|
| ... | ... | |
| 357 | 386 |
private ProtoDocumentMapper initProtoMapper() throws DocumentException, ISLookUpException, ISLookUpDocumentNotFoundException {
|
| 358 | 387 |
return new ProtoDocumentMapper( |
| 359 | 388 |
serviceLocator |
| 360 |
.getService(ISLookUpService.class) |
|
| 361 |
.getResourceProfileByQuery( |
|
| 362 |
"collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + indexFormat
|
|
| 363 |
+ "']//LAYOUT[@name='index']/FIELDS")); |
|
| 389 |
.getService(ISLookUpService.class)
|
|
| 390 |
.getResourceProfileByQuery(
|
|
| 391 |
"collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + indexFormat
|
|
| 392 |
+ "']//LAYOUT[@name='index']/FIELDS"));
|
|
| 364 | 393 |
} |
| 365 | 394 |
|
| 366 | 395 |
private Iterable<Oaf> markDeleted(final Iterable<Oaf.Builder> builders) {
|
| modules/dnet-deduplication/trunk/src/main/resources/eu/dnetlib/web/resources/html/dedup/recordTable.html | ||
|---|---|---|
| 37 | 37 |
<span class="glyphicon glyphicon-minus" aria-hidden="true"></span> |
| 38 | 38 |
</button> |
| 39 | 39 |
<button class="btn btn-primary btn-sm btn-warning" |
| 40 |
ng-click="removeFromGroupAndDeassociateFn(r.id, r.idList, group.group)"
|
|
| 41 |
ng-show="containsList(r.idList, group.group) && removeFromGroupAndDeassociateFn">
|
|
| 40 |
ng-click="addToRemovalGroupFn(r.id)"
|
|
| 41 |
ng-show="!containsList(r.id, removal) && addToRemovalGroupFn">
|
|
| 42 | 42 |
<span class="glyphicon glyphicon-minus" aria-hidden="true"></span> |
| 43 | 43 |
</button> |
| 44 |
<button class="btn btn-primary btn-sm" |
|
| 45 |
ng-click="removeFromRemovalGroupFn(r.id)" |
|
| 46 |
ng-show="containsList(r.id, removal) && addToRemovalGroupFn"> |
|
| 47 |
<span class="glyphicon glyphicon-plus" aria-hidden="true"></span> |
|
| 48 |
</button> |
|
| 44 | 49 |
</td> |
| 45 | 50 |
|
| 46 | 51 |
</tr> |
| modules/dnet-deduplication/trunk/src/main/resources/eu/dnetlib/web/resources/html/dedup/add.html | ||
|---|---|---|
| 92 | 92 |
<ng-dedup-records-table ng-show="group.entityType.type == 'result'" |
| 93 | 93 |
theads="Group size,Title,Publication date,Abstract" |
| 94 | 94 |
fields="groupSize,title,dateofacceptance,description" |
| 95 |
results="group.detailList" group="group" |
|
| 96 |
remove-from-group-and-deassociate-fn="removeAndDeassociate"></ng-dedup-records-table>
|
|
| 95 |
results="group.detailList" group="group" removal="removalGroup"
|
|
| 96 |
add-to-removal-group-fn="addToRemovalGroup" remove-from-removal-group-fn="removeFromRemovalGroup"></ng-dedup-records-table>
|
|
| 97 | 97 |
<ng-dedup-records-table |
| 98 | 98 |
ng-show="group.entityType.type == 'organization'" |
| 99 | 99 |
theads="Group size,Legal name,Legal short name,Website URL" |
| 100 | 100 |
fields="groupSize,legalname,legalshortname,websiteurl" |
| 101 |
results="group.detailList" group="group" |
|
| 102 |
remove-from-group-and-deassociate-fn="removeAndDeassociate"></ng-dedup-records-table>
|
|
| 101 |
results="group.detailList" group="group" removal="removalGroup"
|
|
| 102 |
add-to-removal-group-fn="addToRemovalGroup" remove-from-removal-group-fn="removeFromRemovalGroup"></ng-dedup-records-table>
|
|
| 103 | 103 |
<ng-dedup-records-table |
| 104 | 104 |
ng-show="group.entityType.type == 'person'" |
| 105 | 105 |
theads="Group size,Fullname" |
| 106 | 106 |
fields="groupSize,fullname" |
| 107 |
results="group.detailList" group="group" |
|
| 108 |
remove-from-group-and-deassociate-fn="removeAndDeassociate"></ng-dedup-records-table>
|
|
| 107 |
results="group.detailList" group="group" removal="removalGroup"
|
|
| 108 |
add-to-removal-group-fn="addToRemovalGroup" remove-from-removal-group-fn="removeFromRemovalGroup"></ng-dedup-records-table>
|
|
| 109 | 109 |
</div> |
| 110 | 110 |
</div> |
| 111 | 111 |
</div> |
| 112 | 112 |
|
| 113 | 113 |
<div class="modal-footer"> |
| 114 |
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button> |
|
| 114 |
<button type="button" class="btn btn-default" data-dismiss="modal" ng-click="removeAndDeassociate(group.group)">Apply</button> |
|
| 115 |
<button type="button" class="btn btn-default" data-dismiss="modal" ng-click="dropRemovalGroup()">Undo</button> |
|
| 115 | 116 |
</div> |
| 116 | 117 |
</div> |
| 117 | 118 |
</div> |
| modules/dnet-deduplication/trunk/src/main/resources/eu/dnetlib/web/resources/js/dedup_inspector.js | ||
|---|---|---|
| 109 | 109 |
'fields' : '@', |
| 110 | 110 |
'results' : '=', |
| 111 | 111 |
'group' : '=', |
| 112 |
'removal' : '=', |
|
| 112 | 113 |
|
| 113 | 114 |
'showMergedFn' : '=', |
| 114 | 115 |
'addToGroupFn' : '=', |
| 115 | 116 |
'removeFromGroupFn' : '=', |
| 116 |
'removeFromGroupAndDeassociateFn' : '=', |
|
| 117 |
'addToRemovalGroupFn' : '=', |
|
| 118 |
'removeFromRemovalGroupFn' : '=' |
|
| 117 | 119 |
}, |
| 118 | 120 |
templateUrl: '../resources/html/dedup/recordTable.html', |
| 119 | 121 |
link: function(scope, element, attrs, ctrl) {
|
| modules/dnet-deduplication/trunk/src/main/resources/eu/dnetlib/web/resources/js/dedup_inspector_controllers.js | ||
|---|---|---|
| 56 | 56 |
entityType : $scope.validentityTypes[$routeParams.entityType] |
| 57 | 57 |
} |
| 58 | 58 |
groupService.setGroup($scope.group); |
| 59 |
$scope.removalGroup = []; |
|
| 59 | 60 |
}; |
| 60 | 61 |
|
| 61 | 62 |
$scope.group.actionSet = $routeParams.actionSet; |
| ... | ... | |
| 104 | 105 |
group : {},
|
| 105 | 106 |
entityType : $scope.validentityTypes[$routeParams.entityType] |
| 106 | 107 |
}; |
| 108 |
|
|
| 109 |
$scope.removalGroup = []; |
|
| 107 | 110 |
} |
| 108 | 111 |
|
| 109 | 112 |
$scope.search = function() {
|
| ... | ... | |
| 313 | 316 |
$scope.showNotification('removed ' + ids.length + ' element(s)');
|
| 314 | 317 |
} |
| 315 | 318 |
|
| 316 |
$scope.removeAndDeassociate = function(id, idList, group) {
|
|
| 317 |
$scope.remove(id, idList); |
|
| 319 |
$scope.dropRemovalGroup = function() {
|
|
| 320 |
$scope.removalGroup = []; |
|
| 321 |
} |
|
| 322 |
|
|
| 323 |
$scope.removeFromRemovalGroup = function(id) {
|
|
| 324 |
var removal = $scope.removeFromIdList([id], $scope.removalGroup); |
|
| 325 |
$scope.removalGroup = removal; |
|
| 326 |
} |
|
| 327 |
|
|
| 328 |
$scope.addToRemovalGroup = function(id) {
|
|
| 329 |
$scope.removalGroup.push(id); |
|
| 330 |
} |
|
| 331 |
|
|
| 332 |
$scope.removeAndDeassociate = function(group) {
|
|
| 333 |
var dissimilar = []; |
|
| 334 |
angular.forEach(group, function(gid) {
|
|
| 335 |
if (!$scope.contains(gid, $scope.removalGroup)) {
|
|
| 336 |
dissimilar.push(gid); |
|
| 337 |
} |
|
| 338 |
}); |
|
| 339 |
angular.forEach(dissimilar, function(did) {
|
|
| 340 |
$scope.dissimilar.group[did] = $scope.removalGroup; |
|
| 341 |
}); |
|
| 342 |
|
|
| 343 |
angular.forEach($scope.removalGroup, function(rid) {
|
|
| 344 |
$scope.remove(rid, rid); |
|
| 345 |
}); |
|
| 318 | 346 |
|
| 319 |
var dissimilar = $scope.removeFromIdList([id], group); |
|
| 320 |
$scope.dissimilar.group[id] = dissimilar; |
|
| 347 |
|
|
| 348 |
//$scope.remove(id, idList); |
|
| 349 |
//var dissimilar = $scope.removeFromIdList([id], group); |
|
| 350 |
//$scope.dissimilar.group[id] = dissimilar; |
|
| 351 |
|
|
| 352 |
//var newgroup = $scope.removeFromIdList($scope.removalGroup, $scope.group.group); |
|
| 353 |
//$scope.group.group = newgroup; |
|
| 354 |
|
|
| 355 |
$scope.removalGroup = []; |
|
| 321 | 356 |
} |
| 322 |
|
|
| 323 |
$scope.contains = function(idCsv, list) {
|
|
| 357 |
|
|
| 358 |
$scope.intersect = function(list1, list2) {
|
|
| 324 | 359 |
var res = true; |
| 325 |
if (idCsv && list) {
|
|
| 326 |
angular.forEach(idCsv.split(","), function(id) {
|
|
| 327 |
if(jQuery.grep(list, function(val, i) {
|
|
| 360 |
if (list1 && list2) {
|
|
| 361 |
angular.forEach(list1, function(id) {
|
|
| 362 |
if(jQuery.grep(list2, function(val, i) {
|
|
| 328 | 363 |
return val == id; |
| 329 | 364 |
}).length == 0) {
|
| 330 | 365 |
res = false; |
| ... | ... | |
| 335 | 370 |
return false; |
| 336 | 371 |
} |
| 337 | 372 |
return res; |
| 373 |
} |
|
| 374 |
|
|
| 375 |
$scope.contains = function(idCsv, list) {
|
|
| 376 |
return $scope.intersect(idCsv.split(","), list);
|
|
| 338 | 377 |
} |
| 339 | 378 |
|
| 340 | 379 |
$scope.showRootModal = function(idCsv) {
|
Also available in: Unified diff
fixed representatives/dissimilarities management in the dedup index