D-Net

<?xml version="1.0" encoding="UTF-8"?>

<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"

    xmlns:dnet="eu.dnetlib.actionmanager.actions.infopackage.DataciteInfoPackageToHbaseXsltFunctions"

    xmlns:oaf="http://namespace.openaire.eu/oaf"

    xmlns:dri="http://www.driver-repository.eu/namespace/dri"

   	xmlns:date="eu.dnetlib.miscutils.datetime.DateUtils"

    xmlns:dr="http://www.driver-repository.eu/namespace/dr" xmlns:exslt="http://exslt.org/common"

    extension-element-prefixes="exslt" exclude-result-prefixes="xsl dnet exslt oaf dr dri date">

    <xsl:output omit-xml-declaration="yes" indent="yes"/>

    <xsl:param name="trust" select="string('0.9')"/>

    <xsl:param name="provenance" select="string('UNKNOWN')"/>

    <xsl:param name="namespaceprefix" select="string('datacite____')"/>

    <xsl:template match="/*">

		<xsl:variable name="dataInfo" select="/*[local-name() = 'record']/*[local-name() = 'about']/*[local-name() = 'datainfo']"/>

        <xsl:variable name="metadata" select="exslt:node-set(/*[local-name()='record']/*[local-name()='metadata']/*[local-name()='resource'])"/>

        <xsl:variable name="rightNSPrefix">

            <xsl:choose>

                <xsl:when test="not($namespaceprefix)">

                    <xsl:value-of select="//oaf:datasourceprefix"/>

                </xsl:when>

                <xsl:otherwise>

                    <xsl:value-of select="$namespaceprefix"/>

                </xsl:otherwise>

            </xsl:choose>

        </xsl:variable>

        <xsl:choose>

            <xsl:when test="count($metadata) =  0">

                <ACTIONS/>

            </xsl:when>

            <xsl:otherwise>

                <xsl:variable name="originalId" select="//*[local-name() = 'identifier' and ./@identifierType='DOI']"/>

                <xsl:variable name="resultId" select="dnet:oafSimpleId('result', //dri:objIdentifier)"/>

                <xsl:variable name="creators" select="//*[local-name() = 'creator']"/>

                <xsl:variable name="titles" select="//*[local-name() = 'title']"/>

                <xsl:variable name="subjects" select="//*[local-name() = 'subject']"/>

                <xsl:variable name="publisher" select="//*[local-name() = 'publisher']"/>

                <xsl:variable name="descriptions" select="//*[local-name() = 'description']"/>

                <xsl:variable name="dates" select="//*[local-name() = 'date']"/>

                <xsl:variable name="dateaccepted" select="//oaf:dateAccepted" />

                <xsl:variable name="resourceType" select="//*[local-name() = 'resourceType']"/>

                <xsl:variable name="formats" select="//*[local-name() = 'format']"/>

                <xsl:variable name="sizes" select="//*[local-name() = 'size']"/>

                <xsl:variable name="rights" select="//oaf:accessrights" />

                <xsl:variable name="version" select="//*[local-name() = 'version']"/>

                <xsl:variable name="instanceURI"

                    select="concat('http://dx.doi.org','/',//*[local-name() = 'resource']/*[local-name() = 'identifier'])"/>

                <xsl:variable name="hostedbyid"

                    select="dnet:oafSplitId('datasource', //oaf:hostedBy/@id)"/>

                <xsl:variable name="hostedbyname" select="//oaf:hostedBy/@name"/>

                <xsl:variable name="collectedfromid"

                    select="dnet:oafSplitId('datasource', //oaf:collectedFrom/@id)"/>

                <xsl:variable name="collectedfromname" select="//oaf:collectedFrom/@name"/>

                <xsl:variable name="dateOfCollection" select="//dr:dateOfCollection"/>

                <xsl:variable name="language" select="//oaf:language" />

				<xsl:variable name="cobjcategory" select="//dr:CobjCategory" />                

                <ACTIONS>

                    <ACTION targetKey="{$resultId}" targetColumnFamily="result" targetColumn="{concat('update_', date:now())}">

                        <xsl:value-of

                            select="dnet:oafDataCiteResultFromInfoPackage($resultId, $dataInfo, $metadata, $titles, 

                            $subjects, $publisher, $descriptions, $dates, $dateaccepted, $resourceType, 

                            $formats, $sizes, $language, $cobjcategory, $rights, $version, $provenance, $trust, $hostedbyid, $hostedbyname,

                            $collectedfromid, $collectedfromname, $originalId, $instanceURI, $dateOfCollection)"

                        />

                    </ACTION>

                </ACTIONS>

            </xsl:otherwise>

        </xsl:choose>

    </xsl:template>

</xsl:stylesheet>

<?xml version="1.0" encoding="UTF-8"?>

<xsl:stylesheet version="1.0"

	xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:dc="http://purl.org/dc/elements/1.1/"

	xmlns:dr="http://www.driver-repository.eu/namespace/dr" xmlns:dri="http://www.driver-repository.eu/namespace/dri"

	xmlns:oaa="http://namespace.openaire.eu/oaa" xmlns:oaf="http://namespace.openaire.eu/oaf"

	xmlns:dnet="eu.dnetlib.actionmanager.actions.infopackage.DMFInfoPackageToHbaseXsltFunctions"

	xmlns:date="eu.dnetlib.miscutils.datetime.DateUtils"

	xmlns:exslt="http://exslt.org/common" xmlns:action="http://namespace.openaire.eu/action" 

	extension-element-prefixes="exslt"

	exclude-result-prefixes="xsl dc dr dri oaa oaf dnet exslt date">

	<xsl:output omit-xml-declaration="yes" indent="yes" />

	<xsl:param name="trust" select="string('0.9')" />

	<xsl:param name="provenance" select="string('UNKNOWN')" />

	<xsl:param name="namespaceprefix" select="string('unknown_____')" />

	<xsl:template match="/*">

		<xsl:variable name="dataInfo" select="/*[local-name() = 'record']/*[local-name() = 'about']/*[local-name() = 'datainfo']"/>

		<xsl:variable name="dateofcollection" select="//dr:dateOfCollection"/>

		<xsl:variable name="metadata" select="exslt:node-set(//*[local-name()='metadata']/*)" />

		<xsl:variable name="collectedDatasourceid">

			<xsl:choose>

				<xsl:when test="string-length(//oaf:collectedDatasourceid) &gt; 0">

					<xsl:value-of select="//oaf:collectedDatasourceid" />

				</xsl:when>

				<xsl:otherwise>

					<xsl:value-of select="UNKNOWN" />

				</xsl:otherwise>

			</xsl:choose>

		</xsl:variable>

		<xsl:choose>

			<xsl:when test="count($metadata) = 0">

				<ACTIONS />

			</xsl:when>

			<xsl:otherwise>

				<xsl:variable name="objidentifier" select="/record/*[local-name() = 'header']/*[local-name() = 'objIdentifier']" />

				<xsl:variable name="resultId" select="dnet:oafSimpleId('result', $objidentifier)" />

				<xsl:variable name="hostedbyid" select="dnet:oafSplitId('datasource', //oaf:hostedBy/@id)" />

				<xsl:variable name="hostedbyname" select="//oaf:hostedBy/@name" />

				<xsl:variable name="collectedfromid" select="dnet:oafSplitId('datasource', //oaf:collectedFrom/@id)" />

				<xsl:variable name="collectedfromname" select="//oaf:collectedFrom/@name" />		

<!-- 				<xsl:variable name="country" select="substring(//dr:repositoryCountry,  1, 200)" /> -->

<!-- 				<xsl:variable name="accessmode" select="substring(//oaf:accessrights,   1, 200)" /> -->

				<xsl:variable name="result" select="dnet:oafResultFromInfoPackage($resultId, $dataInfo, $provenance, $trust, $hostedbyid, $hostedbyname, $collectedfromid, $collectedfromname, $objidentifier, $dateofcollection, $metadata)" />

				<ACTIONS> 

					<ACTION targetKey="{$resultId}" targetColumnFamily="result" targetColumn="{concat('update_', date:now())}">

						<xsl:value-of select="$result" />

					</ACTION>

				</ACTIONS>

			</xsl:otherwise>

		</xsl:choose>

	</xsl:template>

</xsl:stylesheet>

package eu.dnetlib.pace.clustering;

import org.junit.Before;

import org.junit.Test;

import eu.dnetlib.pace.AbstractProtoPaceTest;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.FieldListImpl;

import eu.dnetlib.pace.model.FieldValueImpl;

import eu.dnetlib.pace.model.MapDocument;

public class BlacklistAwareClusteringCombinerTest extends AbstractProtoPaceTest {

	private Config config;

	@Before

	public void setUp() {

		config = getResultFullConf();

	}

	@Test

	public void testCombine() {

		MapDocument result = result(config, "A", "Dipping in Cygnus X-2 in a multi-wavelength campaign due to absorption of extended ADC emission", "2013");

		FieldListImpl fl = new FieldListImpl();

		fl.add(new FieldValueImpl(Type.String, "desc", "hello world description pipeline"));

		result.getFieldMap().put("desc", fl);

		fl.clear();

		fl.add(new FieldValueImpl(Type.String, "title", "lorem ipsum cabalie qwerty"));

		result.getFieldMap().get("title").add(fl);

		System.out.println(BlacklistAwareClusteringCombiner.filterAndCombine(result, config, config.blacklists()));

	}

}

package eu.dnetlib.data.mapreduce.util;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertNotNull;

import java.util.List;

import org.junit.Test;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.miscutils.functional.xml.IndentXmlString;

public class OafDecoderTest {

	@Test

	public void testAsXml() {

		final OafDecoder decoder = OafTest.embed(OafTest.getResult("50|id_1"), Kind.entity);

		assertNotNull(decoder);

		assertNotNull(decoder.asXml());

		System.out.println(IndentXmlString.apply(decoder.asXml()));

	}

	@Test

	public void testGetFieldValues() {

		final OafDecoder decoder = OafTest.embed(OafTest.getResult("50|id_1"), Kind.entity);

		final String path = "result/metadata/title/value";

		final List<String> titles = decoder.decodeEntity().getFieldValues("title", path);

		assertNotNull(titles);

		assertFalse(titles.isEmpty());

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<xsl:stylesheet version="1.0"

	xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:dc="http://purl.org/dc/elements/1.1/"

	xmlns:dr="http://www.driver-repository.eu/namespace/dr" xmlns:dri="http://www.driver-repository.eu/namespace/dri"

	xmlns:oaa="http://namespace.openaire.eu/oaa" xmlns:oaf="http://namespace.openaire.eu/oaf"

	xmlns:dnet="eu.dnetlib.data.transform.xml.DNetMdStoreDataCiteToHbaseXsltFunctions"

	xmlns:exslt="http://exslt.org/common" extension-element-prefixes="exslt"

	exclude-result-prefixes="xsl dc dr dri oaa oaf dnet exslt">

	<xsl:output omit-xml-declaration="yes" indent="yes" />

	<xsl:template match="/*">

		<xsl:variable name="dataInfo" select="/*[local-name() = 'record']/*[local-name() = 'about']/*[local-name() = 'datainfo']"/>

		<xsl:variable name="dateofcollection" select="//dri:dateOfCollection" />

		<xsl:variable name="metadata"

			select="exslt:node-set(//*[local-name()='metadata']/*)" />

		<xsl:variable name="namespaceprefix">

			<xsl:choose>

				<!-- TODO check namespaceprefix length is 12 -->

				<xsl:when test="string-length(//oaf:datasourceprefix) &gt; 0">

					<xsl:value-of select="//oaf:datasourceprefix" />

				</xsl:when>

				<xsl:otherwise>

					<xsl:value-of select="unknown_____" />

				</xsl:otherwise>

			</xsl:choose>

		</xsl:variable>

		<xsl:choose>

			<xsl:when

				test="count($metadata) = 0 or normalize-space(//oaf:skipRecord)= 'true'">

				<ROWS />

			</xsl:when>

			<xsl:otherwise>

				<xsl:variable name="resultId"

					select="dnet:oafSimpleId('result', //dri:objIdentifier)" />

				<xsl:if test="string-length($resultId) &gt; 0">

					<xsl:variable name="originalid"

						select="concat('',  //*[local-name() = 'resource']/*[local-name()='identifier'])" />

					<xsl:variable name="creators" select="//*[local-name() = 'creator']" />

					<xsl:variable name="titles" select="//*[local-name() = 'title']" />

					<xsl:variable name="subjects" select="//*[local-name() = 'subject']" />

					<xsl:variable name="publisher" select="//*[local-name() = 'publisher']" />

					<xsl:variable name="descriptions" select="//*[local-name() = 'description']" />

					<xsl:variable name="dates" select="//*[local-name() = 'date']" />

					<xsl:variable name="dateaccepted" select="//oaf:dateAccepted" />

					<xsl:variable name="resourceType"

						select="//*[local-name() = 'resourceType']" />

					<xsl:variable name="formats" select="//*[local-name() = 'format']" />

					<xsl:variable name="sizes" select="//*[local-name() = 'size']" />

					<xsl:variable name="rights" select="//oaf:accessrights" />

					<xsl:variable name="version" select="//*[local-name() = 'version']" />

					<xsl:variable name="language" select="//oaf:language" />

					<xsl:variable name="cobjcategory" select="//dr:CobjCategory" />

					<xsl:variable name="instanceURI">

						<xsl:choose>

							<xsl:when

								test="string-length( //*[local-name() = 'resource']/*[local-name()='identifier' and ./@identifierType='DOI']) > 0">

								<xsl:value-of

									select="concat('http://dx.doi.org','/', //*[local-name() = 'resource']/*[local-name()='identifier' and ./@identifierType='DOI']/text())" />

							</xsl:when>

							<xsl:otherwise>

								<xsl:value-of

									select="concat('', //*[local-name() = 'resource']/*[local-name()='identifier' and ./@identifierType='URL'])" />

							</xsl:otherwise>

						</xsl:choose>

					</xsl:variable>

					<xsl:variable name="hostedbyid"

						select="dnet:oafSplitId('datasource', //oaf:hostedBy/@id)" />

					<xsl:variable name="hostedbyname" select="concat('', //oaf:hostedBy/@name)" />

					<xsl:variable name="collectedfromid"

						select="dnet:oafSplitId('datasource', //oaf:collectedFrom/@id)" />

					<xsl:variable name="collectedfromname"

						select="concat('', //oaf:collectedFrom/@name)" />

					<xsl:variable name="dateOfCollection"

						select="concat('', //dri:dateOfCollection)" />

					<xsl:variable name="result"

						select="dnet:oafResult_FromDatacite($resultId, $dataInfo, $metadata, $titles, 

                        $subjects, $publisher, $descriptions, $dates, $dateaccepted, $resourceType, 

                        $formats, $sizes, $language, $cobjcategory, $rights, $version, $hostedbyid, $hostedbyname,

                        $collectedfromid, $collectedfromname, $originalid, $instanceURI, $dateOfCollection)" />

					<ROWS>

						<ROW key="{$resultId}" columnFamily="result">

							<QUALIFIER name="body" type="base64">

								<xsl:value-of select="$result" />

							</QUALIFIER>

						</ROW>

						<xsl:for-each select="//*[local-name() = 'creator']">

							<xsl:variable name="personIdTemp">

								<xsl:choose>

									<xsl:when

										test="string-length(./*[local-name() = 'nameIdentifier']) > 0">

										<xsl:value-of

											select="translate(normalize-space(./*[local-name() = 'nameIdentifier']),' .,','___')" />

									</xsl:when>

									<xsl:otherwise>

										<xsl:value-of

											select="translate(normalize-space(./*[local-name() = 'creatorName']),' .,','___')" />

									</xsl:otherwise>

								</xsl:choose>

							</xsl:variable>

							<xsl:variable name="personId"

								select="dnet:oafId('person', $namespaceprefix, normalize-space($personIdTemp))" />

							<xsl:variable name="originalPersonId"

								select="./*[local-name() = 'nameIdentifier']" />

							<xsl:variable name="position" select="position()" />

							<xsl:if test="string-length($personId) &gt; 0">

								<xsl:variable name="person"

									select="dnet:oafPerson_FromDatacite($personId, $dataInfo, $collectedfromid, $collectedfromname,$originalPersonId, $dateOfCollection ,normalize-space(./*[local-name() = 'creatorName']))" />

								<xsl:variable name="personresult"

									select="dnet:oafPersonResult_Authorship_FromDatacite($personId, $resultId, $position, 'isAuthorOf', $dataInfo)" />

								<xsl:variable name="resultperson"

									select="dnet:oafPersonResult_Authorship_FromDatacite($resultId, $personId, $position, 'hasAuthor', $dataInfo)" />

								<ROW key="{$personId}" columnFamily="person">

									<QUALIFIER name="body" type="base64">

										<xsl:value-of select="$person" />

									</QUALIFIER>

								</ROW>

								<ROW key="{$personId}" columnFamily="personResult_authorship_isAuthorOf">

									<QUALIFIER name="{$resultId}" type="base64">

										<xsl:value-of select="$personresult" />

									</QUALIFIER>

								</ROW>

								<ROW key="{$resultId}" columnFamily="personResult_authorship_hasAuthor">

									<QUALIFIER name="{$personId}" type="base64">

										<xsl:value-of select="$resultperson" />

									</QUALIFIER>

								</ROW>

							</xsl:if>

						</xsl:for-each>

						<xsl:for-each select="//*[local-name()='projectid']">

							<xsl:variable name="projectId"

								select="dnet:oafSplitId('project', normalize-space(.))" />

							<xsl:variable name="resultproject"

								select="dnet:oafResultProject_Outcome_FromDatacite($resultId, $projectId, 'isProducedBy', $dataInfo)" />

							<xsl:variable name="projectresult"

								select="dnet:oafResultProject_Outcome_FromDatacite($projectId, $resultId, 'produces', $dataInfo)" />

							<xsl:if test="string-length($projectId) &gt; 0">

								<ROW key="{$resultId}" columnFamily="resultProject_outcome_isProducedBy">

									<QUALIFIER name="{$projectId}" type="base64">

										<xsl:value-of select="$resultproject" />

									</QUALIFIER>

								</ROW>

								<ROW key="{$projectId}" columnFamily="resultProject_outcome_produces">

									<QUALIFIER name="{$resultId}" type="base64">

										<xsl:value-of select="$projectresult" />

									</QUALIFIER>

								</ROW>

							</xsl:if>

						</xsl:for-each>

						<xsl:for-each select="//*[local-name()='relatedPublication']">

							<!-- relatedDataset ids must be in the openaire format  -->

							<xsl:variable name="publicationId" select="dnet:oafSimpleId('result', normalize-space(./@id))"/>

							<xsl:if test="string-length($publicationId) &gt; 0">

								<xsl:variable name="resultDataset" select="dnet:oafResultResult_PublicationDataset_FromDatacite($resultId, $publicationId, 'isRelatedTo', $dataInfo)"/>

								<xsl:variable name="datasetResult" select="dnet:oafResultResult_PublicationDataset_FromDatacite($publicationId, $resultId, 'isRelatedTo', $dataInfo)"/>

								<ROW key="{$resultId}" columnFamily="resultResult_publicationDataset_isRelatedTo">

									<QUALIFIER name="{$publicationId}" type="base64"><xsl:value-of select="$resultDataset"/></QUALIFIER>

								</ROW>

								<ROW key="{$publicationId}" columnFamily="resultResult_publicationDataset_isRelatedTo">

									<QUALIFIER name="{$resultId}" type="base64"><xsl:value-of select="$datasetResult"/></QUALIFIER>

								</ROW>

							</xsl:if>

						</xsl:for-each>

					</ROWS>

				</xsl:if>

			</xsl:otherwise>

		</xsl:choose>

	</xsl:template>

</xsl:stylesheet>

<?xml version="1.0" encoding="UTF-8"?>

<xsl:stylesheet version="1.0"

	xmlns:xsl="http://www.w3.org/1999/XSL/Transform" 

	xmlns:dnet="eu.dnetlib.data.transform.xml.AbstractDNetOafXsltFunctions"

	exclude-result-prefixes="xsl dnet">

    <xsl:output omit-xml-declaration="yes" indent="yes"/>

	<xsl:template match="/*">

		<xsl:variable name="metadata" select="//*[local-name()='metadata']/*" />

		<ROWS>

			<xsl:value-of select="dnet:testParse($metadata)"/>

		</ROWS>

	</xsl:template>

</xsl:stylesheet>

package eu.dnetlib.data.transform;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertNotNull;

import java.io.IOException;

import java.io.StringWriter;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.common.SolrInputDocument;

import org.apache.solr.common.SolrInputField;

import org.dom4j.DocumentException;

import org.junit.Before;

import org.junit.Test;

import com.google.protobuf.InvalidProtocolBufferException;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.data.mapreduce.util.OafTest;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.proto.OafProtos.OafEntity;

import eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory;

public class ProtoDocumentMapperTest {

	private static final Log log = LogFactory.getLog(ProtoDocumentMapperTest.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String fields;

	@Before

	public void setUp() throws IOException {

		final StringWriter sw = new StringWriter();

		IOUtils.copy(getClass().getResourceAsStream("fields.xml"), sw);

		fields = sw.toString();

		assertNotNull(fields);

		assertFalse(fields.isEmpty());

		log.info(fields);

	}

	@Test

	public void testProto2SolrDocument() throws DocumentException, InvalidProtocolBufferException {

		final ProtoDocumentMapper mapper = new ProtoDocumentMapper(fields);

		assertNotNull(mapper);

		final OafEntity.Builder entity = OafTest.getResultBuilder("01");

		entity.addChildren(OafTest.getResultBuilder("01_children"));

		final Oaf oaf = OafTest.embed(entity.build(), Kind.entity).getOaf();

		assertNotNull(oaf.getEntity().getChildrenList());

		assertFalse(oaf.getEntity().getChildrenList().isEmpty());

		log.info("byte[] size: " + oaf.toByteArray().length);

		log.info("json size:   " + JsonFormat.printToString(oaf).length());

		log.info("base64 size: " + Base64.encodeBase64String(oaf.toByteArray()).length());

		final byte[] decodeBase64 = Base64.decodeBase64(Base64.encodeBase64String(oaf.toByteArray()));

		log.info("decoded: " + JsonFormat.printToString(Oaf.parseFrom(decodeBase64)));

		final SolrInputDocument doc = mapper.map(oaf, InputDocumentFactory.getParsedDateField("2015-02-15"), "asd");

		assertNotNull(doc);

		for (final SolrInputField f : doc.values()) {

			log.info(f);

		}

	}

}

pace.conf { 

	conditions { },

	model { 

		title { algo = JaroWinkler, type = String, weight = 0.5, ignoreMissing = false, path = result/metadata/title/value }

	}

}

package eu.dnetlib.pace.distance;

import static org.junit.Assert.assertTrue;

import java.util.List;

import org.junit.Test;

import com.google.common.collect.Lists;

import eu.dnetlib.pace.AbstractProtoPaceTest;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.model.MapDocument;

public class DetectorTest extends AbstractProtoPaceTest {

	@Test

	public void testDistanceResultSimple() {

		final Config config = getResultSimpleConf();

		final MapDocument resA = result(config, "A", "Recent results from CDF");

		final MapDocument resB = result(config, "B", "Recent results from CDF");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 1.0);

	}

	@Test

	public void testDistanceResultSimpleMissingDates() {

		final Config config = getResultSimpleConf();

		final MapDocument resA = result(config, "A", "Recent results from BES");

		final MapDocument resB = result(config, "A", "Recent results from CES");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d > 0.97);

	}

	@Test

	public void testDistanceResultInvalidDate() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "title title title 6BESR", "2013-01-05");

		final MapDocument resB = result(config, "B", "title title title 6BESR", "qwerty");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 1.0);

	}

	@Test

	public void testDistanceResultMissingOneDate() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "title title title 6BESR", null);

		final MapDocument resB = result(config, "B", "title title title 6CLER", "2012-02");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue((d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceResult() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "title title title BES", "");

		final MapDocument resB = result(config, "B", "title title title CLEO");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		// assertTrue(d > 0.9 && d < 1.0);

	}

	@Test

	public void testDistanceResultMissingTwoDate() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "title title title 6BESR");

		final MapDocument resB = result(config, "B", "title title title 6CLER");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue((d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceOrganizationIgnoreMissing() {

		final Config config = getOrganizationSimpleConf();

		final MapDocument orgA = organization(config, "A", "CONSIGLIO NAZIONALE DELLE RICERCHE");

		final MapDocument orgB = organization(config, "B", "CONSIGLIO NAZIONALE DELLE RICERCHE", "CNR");

		final double d = new PaceDocumentDistance().between(orgA, orgB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 1.0);

	}

	@Test

	public void testDistanceResultCase1() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003");

		final MapDocument resB = result(config, "B", "Search for the Standard Model Higgs Boson", "2003");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue((d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceResultCaseDoiMatch1() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003", "http://dx.doi.org/10.1594/PANGAEA.726855");

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", "10.1594/PANGAEA.726855");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("exact DOIs will produce an exact match", d == 1.0);

	}

	@Test

	public void testDistanceResultCaseDoiMatch2() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "http://dx.doi.org/10.1594/PANGAEA.726855");

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2005", "doi:10.1594/PANGAEA.726855");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("exact DOIs will produce an exact match, regardless of different titles or publication years", d == 1.0);

	}

	@Test

	public void testDistanceResultCaseDoiMatch3() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024");

		final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("a missing DOI will casue the comparsion to continue with the following conditions", d == 1.0);

	}

	@Test

	public void testDistanceResultCaseDoiMatch4() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024");

		final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2005");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("a missing DOI, comparsion continues with the following conditions, different publication years will drop the score to 0", d == 0.0);

	}

	@Test

	public void testDistanceResultCaseDoiMatch5() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Search for the Standard Model Higgs Boson", "2003", "10.1016/j.jmb.2010.12.020");

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("a missing DOI, comparsion continues with the following conditions", (d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceResultCaseDoiMatch6() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024");

		final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003", "anotherDifferentDOI");

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("different DOIs will drop the score to 0, regardless of the other fields", d == 0.0);

	}

	// http://dx.doi.org/10.1594/PANGAEA.726855 doi:10.1594/PANGAEA.726855

	@Test

	public void testDistanceResultCaseAuthor1() {

		final Config config = getResultAuthorsConf();

		final List<String> authorsA = Lists.newArrayList("a", "b", "c", "d");

		final List<String> authorsB = Lists.newArrayList("a", "b", "c");

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", null, authorsA);

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", null, authorsB);

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 0.0);

	}

	@Test

	public void testDistanceResultCaseAuthor2() {

		final Config config = getResultAuthorsConf();

		final List<String> authorsA = Lists.newArrayList("a", "b", "c");

		final List<String> authorsB = Lists.newArrayList("a", "b", "c");

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", null, authorsA);

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", null, authorsB);

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 1.0);

	}

	@Test

	public void testDistanceResultCaseAuthor3() {

		final Config config = getResultAuthorsConf();

		final List<String> authorsA = Lists.newArrayList("Bardi, A.", "Manghi, P.", "Artini, M.");

		final List<String> authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele");

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", null, authorsA);

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", null, authorsB);

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		assertTrue((d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceResultCaseAuthor4() {

		final Config config = getResultAuthorsConf();

		final List<String> authorsA = Lists.newArrayList("Bardi, Alessia", "Manghi, Paolo", "Artini, Michele", "a");

		final List<String> authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele");

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", null, authorsA);

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", null, authorsB);

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		// assertTrue(d == 0.0);

	}

	@Test

	public void testDistanceResultFullConf() {

		final Config config = getResultFullConf();

		final List<String> authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva");

		final List<String> authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie");

		final MapDocument resA =

				result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010",

						"10.1186/1752-1947-4-299", authorsA);

		final MapDocument resB =

				result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010", null,

						authorsB);

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		// assertTrue(d == 0.0);

	}

	@Test

	public void testDistanceStdl1() {

		final Config config = getResultStdlConf();

		final List<String> authorsA = Lists.newArrayList();

		final List<String> authorsB =

				Lists.newArrayList("Giovannelli A.", "Traversi M. L.", "Sebastiani L.", "Tognetti R.", "d?Andria R.", "Morelli G.", "Fragnito F.", "Lavini A.");

		final MapDocument resA =

				result(config,

						"A",

						"Evaluating water use strategies in olive trees grown under different irrigation regimes through integrated approaches between sap flow and high resolution stem growth analysis",

						"2008-01-01",

						null, authorsA);

		final MapDocument resB =

				result(config,

						"B",

						"Evaluating water use strategies in olive trees grown under different irrigation regimes through integrated approaches between sap flow and high resolution stem growth analysis",

						"2008-01-01",

						null,

						authorsB);

		final double d = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(String.format(" d ---> %s", d));

		// assertTrue(d == 0.0);

	}

}

pace.conf { 

	clustering { 

		acronyms { fields = [title, desc], params = { max = 1, minLen = 2, maxLen = 4} }, 

		ngrampairs { fields = [title], params = { max = 1, ngramLen = 3} }, 

		suffixprefix { fields = [title], params = { max = 1, len = 3 } }

	}, 

	conditions { 

		titleVersionMatch { fields = [title] },

		sizeMatch { fields = [authors] }  

	},

	model { 

		title { algo = JaroWinkler, type = String, weight = 0.5, ignoreMissing = false, path = result/metadata/title/value },

		authors { algo = SortedLevel2JaroWinkler, type = List, weight = 0.5, ignoreMissing = true, path = result/author/metadata/fullname/value } 		

	}, 

	blacklists = {

		title = [

			"^(Corpus Oral Dialectal \\(COD\\)\\.).*$",

			"^(Kiri Karl Morgensternile).*$",

			"^(\\[Eksliibris Aleksandr).*\\]$",

			"^(\\[Eksliibris Aleksandr).*$",

			"^(Eksliibris Aleksandr).*$",

			"^(Kiri A\\. de Vignolles).*$",

			"^(2 kirja Karl Morgensternile).*$",

			"^(Pirita kloostri idaosa arheoloogilised).*$",

			"^(Kiri tundmatule).*$",

			"^(Kiri Jenaer Allgemeine Literaturzeitung toimetusele).*$",

			"^(Eksliibris Nikolai Birukovile).*$",

			"^(Eksliibris Nikolai Issakovile).*$",

			"^(WHP Cruise Summary Information of section).*$",

			"^(Measurement of the top quark\\-pair production cross section with ATLAS in pp collisions at).*$",

			"^(Measurement of the spin\\-dependent structure function).*"

		] }

}

package eu.dnetlib.pace.model;

import org.junit.Test;

import eu.dnetlib.pace.AbstractProtoPaceTest;

import eu.dnetlib.pace.config.Config;

public class ProtoDocumentBuilderTest extends AbstractProtoPaceTest {

	@Test

	public void test1() {

		String id = "12345";

		Config config = getResultConf();

		MapDocument document = ProtoDocumentBuilder.newInstance(id, getResult(id), config.fields());

		System.out.println(document);

		String stringDoc = MapDocumentSerializer.toString(document);

		System.out.println(stringDoc);

		MapDocument decoded = MapDocumentSerializer.decode(stringDoc.getBytes());

		System.out.println(decoded);

	}

}

package eu.dnetlib.data.transform;

import java.util.List;

import java.util.Map;

import java.util.Set;

import com.google.common.base.Predicate;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import com.google.common.collect.Sets;

import com.google.protobuf.Descriptors.FieldDescriptor;

import com.google.protobuf.Message.Builder;

import eu.dnetlib.data.proto.FieldTypeProtos.KeyValue;

import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;

import eu.dnetlib.data.proto.FieldTypeProtos.StringField;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.proto.OafProtos.OafEntity;

import eu.dnetlib.data.proto.PersonProtos.Person;

import eu.dnetlib.data.proto.ResultProtos.Result;

import eu.dnetlib.data.proto.SpecialTrustProtos.SpecialTrust;

import eu.dnetlib.pace.util.DedupConfig;

public class OafEntityMerger {

	private static final String DEDUP_CLASSID = "sysimport:dedup";

	private static final String DNET_PROVENANCE_SCHEME = "dnet:provenanceActions";

	private final Predicate<StringField> skipEmptyStringField = new Predicate<StringField>() {

		@Override

		public boolean apply(final StringField s) {

			return (s != null) && (s.getValue() != null) && !s.getValue().isEmpty();

		}

	};

	private final Predicate<String> skipEmptyString = new Predicate<String>() {

		@Override

		public boolean apply(final String s) {

			return (s != null) && !s.isEmpty();

		}

	};

	public static Oaf.Builder merge(final String id, final Iterable<Oaf> entities) {

		return merge(null, id, entities);

	}

	public static Oaf.Builder merge(final DedupConfig dedupConf, final String id, final Iterable<Oaf> entities) {

		return new OafEntityMerger().mergeEntities(dedupConf, id, entities);

	}

	public static Oaf.Builder merge(final Oaf.Builder builder) {

		return new OafEntityMerger().doMergeEntities(builder);

	}

	public Oaf.Builder mergeEntities(final DedupConfig dedupConf, final String id, final Iterable<Oaf> entities) {

		Oaf.Builder builder = Oaf.newBuilder();

		String trust = "0.0";

		for (final Oaf oaf : TrustOrdering.sort(entities)) {

			// doublecheck we're dealing only with main entities

			if (!oaf.getKind().equals(Kind.entity)) throw new IllegalArgumentException("expected OafEntity!");

			final String currentTrust = oaf.getDataInfo().getTrust();

			if (!currentTrust.equals(SpecialTrust.NEUTRAL.toString())) {

				trust = currentTrust;

			}

			builder.mergeFrom(oaf);

		}

		builder = doMergeEntities(builder);

		builder.getEntityBuilder().setId(id);

		builder.getDataInfoBuilder()

				.setInferred(true)

				.setDeletedbyinference(false)

				.setTrust(trust)

				.setProvenanceaction(getProvenanceAction());

		if ((dedupConf != null) && dedupConf.isIncludeChildren()) {

			for (final Oaf oaf : Iterables.limit(entities, dedupConf.getMaxChildren())) {

				builder.getEntityBuilder().addChildren(oaf.getEntity());

			}

		}

		return builder;

	}

	private Qualifier.Builder getProvenanceAction() {

		return Qualifier.newBuilder().setClassid(DEDUP_CLASSID).setClassname(DEDUP_CLASSID).setSchemeid(DNET_PROVENANCE_SCHEME)

				.setSchemename(DNET_PROVENANCE_SCHEME);

	}

	public Oaf.Builder doMergeEntities(final Oaf.Builder builder) {

		switch (builder.getEntity().getType()) {

		case datasource:

			break;

		case organization:

			break;

		case person:

			final Person.Metadata.Builder person = builder.getEntityBuilder().getPersonBuilder().getMetadataBuilder();

			for (final String field : Lists.newArrayList("secondnames")) {

				setSingleString(person, field);

			}

			break;

		case project:

			break;

		case result:

			final Result.Metadata.Builder result = builder.getEntityBuilder().getResultBuilder().getMetadataBuilder();

			setTitle(result);

			// for (String field : Lists.newArrayList("subject", "relevantdate")) {

			for (final String field : OafUtils.getFieldNames(Result.Metadata.getDescriptor(), Result.Metadata.SUBJECT_FIELD_NUMBER,

					Result.Metadata.RELEVANTDATE_FIELD_NUMBER)) {

				setStructuredProperty(result, field);

			}

			for (final String field : OafUtils.getFieldNames(Result.Metadata.getDescriptor(), Result.Metadata.DESCRIPTION_FIELD_NUMBER)) {

				setLongestStringField(result, field);

			}

			for (final String field : OafUtils.getFieldNames(Result.Metadata.getDescriptor(), Result.Metadata.SOURCE_FIELD_NUMBER)) {

				setUniqueStringField(result, field);

			}

			for (final String field : OafUtils.getFieldNames(OafEntity.getDescriptor(), OafEntity.COLLECTEDFROM_FIELD_NUMBER)) {

				setKeyValues(builder.getEntityBuilder(), field);

			}

			for (final String field : OafUtils.getFieldNames(OafEntity.getDescriptor(), OafEntity.PID_FIELD_NUMBER)) {

				setStructuredProperty(builder.getEntityBuilder(), field);

			}

			for (final String field : OafUtils.getFieldNames(OafEntity.getDescriptor(), OafEntity.ORIGINALID_FIELD_NUMBER)) {

				setUniqueString(builder.getEntityBuilder(), field);

			}

			// remove the inner authors, rely on the children

			builder.getEntityBuilder().getResultBuilder().clearAuthor();

			break;

		default:

			break;

		}

		return builder;

	}

	/**

	 * Helper method, avoid duplicated StructuredProperties in the given builder for the given fieldName

	 *

	 * @param builder

	 * @param fieldName

	 */

	@SuppressWarnings("unchecked")

	private void setStructuredProperty(final Builder builder, final String fieldName) {

		final Map<String, StructuredProperty> map = Maps.newHashMap();

		final FieldDescriptor fd = builder.getDescriptorForType().findFieldByName(fieldName);

		final List<StructuredProperty> sps = (List<StructuredProperty>) builder.getField(fd);

		if ((sps != null) && !sps.isEmpty()) {

			for (final StructuredProperty sp : sps) {

				map.put(sp.getValue(), sp);

			}

			if (!map.isEmpty()) {

				builder.clearField(fd).setField(fd, Lists.newArrayList(map.values()));

			}

		}

	}

	/**

	 * Helper method, avoid duplicated KeyValues in the given builder for the given fieldName

	 *

	 * @param builder

	 * @param fieldName

	 */

	@SuppressWarnings("unchecked")

	private void setKeyValues(final Builder builder, final String fieldName) {

		final Map<String, KeyValue> map = Maps.newHashMap();

		final FieldDescriptor fd = builder.getDescriptorForType().findFieldByName(fieldName);

		final List<KeyValue> kvs = (List<KeyValue>) builder.getField(fd);

		if ((kvs != null) && !kvs.isEmpty()) {

			for (final KeyValue sp : kvs) {

				map.put(sp.getKey(), sp);

			}

			if (!map.isEmpty()) {

				builder.clearField(fd).setField(fd, Lists.newArrayList(map.values()));

			}

		}

	}

	@SuppressWarnings("unchecked")

	private void setSingleString(final Builder builder, final String fieldName) {

		final FieldDescriptor fd = builder.getDescriptorForType().findFieldByName(fieldName);

		final List<StringField> field = (List<StringField>) builder.getField(fd);

		if ((field != null) && !field.isEmpty()) {

			final StringField s = (StringField) Iterables.getLast(Iterables.filter(field, skipEmptyStringField), "");

			if ((s != null) && (s.getValue() != null) && !s.getValue().isEmpty()) {

				builder.clearField(fd).setField(fd, Lists.newArrayList(s));

			}

		}

	}

	@SuppressWarnings("unchecked")

	private void setLongestStringField(final Builder builder, final String fieldName) {

		final FieldDescriptor fd = builder.getDescriptorForType().findFieldByName(fieldName);

		final List<StringField> field = (List<StringField>) builder.getField(fd);

		if ((field != null) && !field.isEmpty()) {

			final StringField.Builder max = StringField.newBuilder().setValue("");

			int maxLength = 0;

			for (final StringField sf : field) {

				if (sf.getValue().length() > maxLength) {

					maxLength = sf.getValue().length();

					max.clear();

					max.mergeFrom(sf);

				}

			}

			builder.clearField(fd).setField(fd, Lists.newArrayList(max.build()));

		}

	}

	@SuppressWarnings("unchecked")

	private void setUniqueStringField(final Builder builder, final String fieldName) {

		final FieldDescriptor fd = builder.getDescriptorForType().findFieldByName(fieldName);

		final List<StringField> field = (List<StringField>) builder.getField(fd);

		final Map<String, StringField> map = Maps.newHashMap();

		if ((field != null) && !field.isEmpty()) {

			for (final StringField s : Iterables.filter(field, skipEmptyStringField)) {

				map.put(s.getValue(), s);

			}

			builder.clearField(fd).setField(fd, Lists.newArrayList(map.values()));

		}

	}

	@SuppressWarnings("unchecked")

	private void setUniqueString(final Builder builder, final String fieldName) {

		final FieldDescriptor fd = builder.getDescriptorForType().findFieldByName(fieldName);

		final List<String> field = (List<String>) builder.getField(fd);

		final Set<String> set = Sets.newHashSet();

		if ((field != null) && !field.isEmpty()) {

			for (final String s : Iterables.filter(field, skipEmptyString)) {

				set.add(s);

			}

			builder.clearField(fd).setField(fd, Lists.newArrayList(set));

		}

	}

	private void setTitle(final Result.Metadata.Builder metadata) {

		final Iterable<StructuredProperty> filtered = Iterables.filter(metadata.getTitleList(), OafUtils.mainTitleFilter());

		if (!Iterables.isEmpty(filtered)) {

			metadata.clearTitle().addTitle(Iterables.getLast(filtered));

		}

	}

}

package eu.dnetlib.data.transform;

import java.io.StringReader;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.lang.StringUtils;

import org.apache.solr.common.SolrInputDocument;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Element;

import org.dom4j.io.SAXReader;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

import com.google.protobuf.GeneratedMessage;

/**

 * The Class ProtoDocumentMapper.

 */

public class ProtoDocumentMapper extends AbstractProtoMapper {

	/** The fields. */

	private Document fields;

	/**

	 * Instantiates a new proto document mapper.

	 *

	 * @param fields

	 *            the fields

	 * @throws DocumentException

	 *             the document exception

	 */

	public ProtoDocumentMapper(final String fields) throws DocumentException {

		this.fields = parse(fields);

		if (StringUtils.isBlank(this.fields.valueOf("//FIELD[@name = 'objIdentifier']/@name")))

			throw new IllegalArgumentException("field objIdentifier is mandatory");

	}

	/**

	 * Map.

	 *

	 * @param proto

	 *            the proto

	 * @param version

	 *            the version

	 * @param dsId

	 *            the ds id

	 * @return the solr input document

	 * @throws DocumentException

	 *             the document exception

	 */

	public SolrInputDocument map(final GeneratedMessage proto, final String version, final String dsId) throws DocumentException {

		final SolrInputDocument doc = new SolrInputDocument();

		for (final Object o : fields.selectNodes("//FIELD")) {

			final Element e = (Element) o;

			final String name = e.attribute("name").getValue().toLowerCase().trim();

			final String path = e.attribute("path").getValue();

			doc.setField(name, processMultiPath(proto, Lists.newLinkedList(Splitter.on("|").trimResults().split(path))));

		}

		doc.setField("__dsid", dsId);

		doc.setField("__dsversion", version);

		doc.setField("objidentifier", patchId((String) doc.getFieldValue("objidentifier")));

		doc.setField("__indexrecordidentifier", doc.getFieldValue("objidentifier"));

		doc.setField("__result", Base64.encodeBase64String(proto.toByteArray()));

		return doc;

	}

	/**

	 * Patch the objidentifier: when it comes from HBase, i.e. contains the separator '|' returns the string that follows.

	 *

	 * @param objidentifier

	 *            the objidentifier

	 * @return the string

	 */

	private String patchId(final String objidentifier) {

		return objidentifier.contains("|") ? StringUtils.substringAfter(objidentifier, "|") : objidentifier;

	}

	/**

	 * Parses the.

	 *

	 * @param s

	 *            the s

	 * @return the document

	 * @throws DocumentException

	 *             the document exception

	 */

	private Document parse(final String s) throws DocumentException {

		return new SAXReader().read(new StringReader(s));

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-hadoop-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-openaireplus-mapping-utils</artifactId>

	<packaging>jar</packaging>

	<version>3.0.7-SNAPSHOT</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-openaireplus-mapping-utils/trunk</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>		

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>commons-codec</groupId>

			<artifactId>commons-codec</artifactId>

			<version>${commons.codec.version}</version>

		</dependency>

		<dependency>

			<groupId>dom4j</groupId>

			<artifactId>dom4j</artifactId>

			<version>${dom4j.version}</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaire-data-protos</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-pace-core</artifactId>

			<version>[1.3.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-misc-utils</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-hadoop-commons</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-index-solr-common</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>	

		<dependency>

			<groupId>com.googlecode.protobuf-java-format</groupId>

			<artifactId>protobuf-java-format</artifactId>

			<version>1.2</version>

			<scope>test</scope>

		</dependency>		

	</dependencies>

</project>

package eu.dnetlib.data.mapreduce.util;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import org.junit.Before;

import org.junit.Test;

import com.google.protobuf.Descriptors.FieldDescriptor;

import eu.dnetlib.data.proto.OafProtos.OafRel;

import eu.dnetlib.data.proto.PersonResultProtos.PersonResult.Authorship;

import eu.dnetlib.data.proto.RelTypeProtos.RelType;

import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;

public class OafRelDecoderTest {

	private OafRel oafRel;

	@Before

	public void setUp() {

		oafRel = OafTest.getPersonResult("ID_1", "ID_2", "1", "isAuthor");

	}

	@Test

	public void testSetClass() {

		OafRelDecoder d1 = OafRelDecoder.decode(oafRel);

		assertNotNull(d1);

		assertEquals("isAuthor", d1.getRelClass());

		OafRelDecoder d2 = OafRelDecoder.decode(d1.setClassId("hasAuthor").build());

		assertEquals("hasAuthor", d2.getRelClass());

		assertEquals("hasAuthor", d2.getRelMetadata().getSemantics().getClassid());

		assertEquals("hasAuthor", d2.getRelMetadata().getSemantics().getClassname());

		FieldDescriptor fd = Authorship.getDescriptor().findFieldByName("ranking");

		assertEquals(d1.getSubRel().getField(fd), d2.getSubRel().getField(fd));

	}

	@Test

	public void testGetCF() {

		assertEquals("personResult_authorship_isAuthorOf", OafRelDecoder.getCFQ(RelType.personResult, SubRelType.authorship, Authorship.RelName.isAuthorOf));

		assertEquals("personResult_authorship_isAuthorOf", OafRelDecoder.getCFQ(RelType.personResult, SubRelType.authorship, "isAuthorOf"));

	}

}

package eu.dnetlib.data.transform;

import java.util.List;

import org.apache.commons.lang.StringUtils;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

import com.google.protobuf.Descriptors.EnumValueDescriptor;

import com.google.protobuf.Descriptors.FieldDescriptor;

import com.google.protobuf.GeneratedMessage;

/**

 * AbstractProtoMapper provide common navigation methods on the protocolbuffers Messages.

 *

 * @author claudio

 */

public abstract class AbstractProtoMapper {

	/** The Constant PATH_SEPARATOR. */

	private static final String PATH_SEPARATOR = "/";

	/**

	 * Process multi path.

	 *

	 * @param proto

	 *            the proto

	 * @param paths

	 *            the paths

	 * @return the list

	 */

	protected List<Object> processMultiPath(final GeneratedMessage proto, final List<String> paths) {

		final List<Object> response = Lists.newArrayList();

		for (final String pathElements : paths) {

			response.addAll(processPath(proto, pathElements));

		}

		return response;

	}

	/**

	 * Process path.

	 *

	 * @param proto

	 *            the proto

	 * @param path

	 *            the path

	 * @return the list

	 */

	protected List<Object> processPath(final GeneratedMessage proto, final String path) {

		return processPath(proto, Lists.newLinkedList(Splitter.on(PATH_SEPARATOR).trimResults().split(path)));

	}