Index: SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/RDFSchemaAnalyzer.java =================================================================== --- SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/RDFSchemaAnalyzer.java (revision 267) +++ SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/RDFSchemaAnalyzer.java (revision 269) @@ -27,9 +27,9 @@ * @since 28.01.2014 */ - public SClass[] getOWLClasses(String[] graphURIs, String[] keyword, String language, boolean countInstances) throws Exception; + public SClass[] getOWLClasses(String graphURI, String[] keyword, String language, boolean countInstances) throws Exception; - public List getOWLClassList(String[] graphURIs, String[] keyword, String language, boolean countInstances) throws Exception; + public List getOWLClassList(String graphURI, String[] keyword, String language, boolean countInstances) throws Exception; - public SClass[] listClasses(String[] graphURIs, boolean countInstances) throws Exception; + public SClass[] listClasses(String graphURI, boolean countInstances) throws Exception; /** @@ -66,5 +66,5 @@ * @since 28.01.2014 */ - public ClassLink[] getNextClass(String[] graphURIs, String originClass, int limit, boolean countLinks) throws Exception; + public ClassLink[] getNextClass(String graphURI, String originClass, int limit, boolean countLinks) throws Exception; /** @@ -111,5 +111,5 @@ // int limit) throws Exception; - public LabelMap[] getLabels(String[] graphURIs, String[] resourceURIs, String language) throws Exception; + public LabelMap[] getLabels(String graphURI, String[] resourceURIs, String language) throws Exception; // public ClassLink[] countLinks(String[] graphURIs, String startClassURI, Index: SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/StructureCrawler.java =================================================================== --- SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/StructureCrawler.java (revision 267) +++ SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/StructureCrawler.java (revision 269) @@ -6,7 +6,7 @@ import java.util.Set; -import jp.riken.accc.db.rdf.crawler.dataStructure.SchemaCategory; -import jp.riken.accc.db.rdf.crawler.dataStructure.sparql.JenaModelGenerator; -import jp.riken.accc.db.rdf.crawler.dataStructure.sparql.RDFsCrawlerImpl; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.SchemaCategory; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.JenaModelGenerator; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.RDFsCrawlerImpl; public class StructureCrawler { @@ -75,20 +75,6 @@ } - public void crawl(String endPURI, String outFileName) throws Exception { - RDFsCrawlerImpl impl = new RDFsCrawlerImpl(endPURI); - // Resource[] res = impl.getRDFProperties(); - // Resource[] res = impl.getInferedRDFsClassesFromInstances(); - // Resource[] res = impl.getDomainRangeDeclaredRDFProperties(); - // Resource[] res = impl.getDeclaredRDFsClasses(); - // for(Resource r: res){ - // System.out.println(r.getURI().toString()); - // } - // Model model = impl.getProperiesFromDomainRangeDecls(); - // Model model = impl.getPropertiesFromInstanceDecls(); - // RDFWriter writer = model.getWriter("N3"); - // writer.setProperty("showXMLDeclaration","true"); - // writer.write(model,System.out,""); - // model.close(); - + public void crawl(String endPURI, String crawlName, String outFileName) throws Exception { + RDFsCrawlerImpl impl = new RDFsCrawlerImpl(endPURI, crawlName); SchemaCategory sc = impl.determineSchemaCategory(); Index: SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/AcquiredStructureAnalyzer.java =================================================================== --- SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/AcquiredStructureAnalyzer.java (revision 267) +++ SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/AcquiredStructureAnalyzer.java (revision 269) @@ -3,6 +3,8 @@ import java.util.*; -import jp.riken.accc.db.rdf.crawler.dataStructure.sparql.JenaModelGenerator; -import jp.riken.accc.db.rdf.crawler.dataStructure.sparql.URICollection; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.JenaModelGenerator; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.URICollection; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.crawler.CrawledMetadata; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.crawler.Dataset; import com.hp.hpl.jena.query.Query; @@ -20,14 +22,12 @@ public class AcquiredStructureAnalyzer implements RDFSchemaAnalyzer { - private Model model = null; - private String endpointURI = null; - private String[] graphURIs = null; + private CrawledMetadata crawledMetadata = null; public String getEndpointURI(){ - return endpointURI; + return crawledMetadata.getEndpointURI(); } public String[] getGraphURIs(){ - return graphURIs; + return crawledMetadata.getGraphURIs(); } @@ -57,30 +57,31 @@ */ - public AcquiredStructureAnalyzer(String endpointURI, String[] graphURIs, Model model){ - this.model = model; - this.endpointURI = endpointURI; - this.graphURIs = graphURIs; - } - - private String[] filterGraphURIs(String[] orgGraphURIs){ + public AcquiredStructureAnalyzer(CrawledMetadata crawledMetadata){ + this.crawledMetadata = crawledMetadata; + } + + private String filterGraphURIs(String orgGraphURIs){ // TODO - return graphURIs; - } - - - public SClass[] listClasses(String[] graphURIs, boolean countInstances) throws Exception{ - return getOWLClasses(graphURIs, null, null, countInstances); - } - - - - - public SClass[] getOWLClasses(String[] graphURIs, String[] keywords, String language, boolean countInstances) throws Exception{ - return getOWLClassList(graphURIs, keywords, language, countInstances).toArray(new SClass[0]); - } - - public List getOWLClassList(String[] graphURIs, String[] keywords, String language, boolean countInstances) throws Exception{ - String[] targetGraphURIs = filterGraphURIs(graphURIs); - + return orgGraphURIs; + } + + + public SClass[] listClasses(String graphURI, boolean countInstances) throws Exception{ + return getOWLClasses(graphURI, null, null, countInstances); + } + + + + + public SClass[] getOWLClasses(String graphURI, String[] keywords, String language, boolean countInstances) throws Exception{ + return getOWLClassList(graphURI, keywords, language, countInstances).toArray(new SClass[0]); + } + + public List getOWLClassList(String graphURI, String[] keywords, String language, boolean countInstances) throws Exception{ + String targetGraphURI = filterGraphURIs(graphURI); + + Dataset dataset = crawledMetadata.getDataset(targetGraphURI); + + StringBuffer queryStr = new StringBuffer(); queryStr.append("PREFIX owl: \n"); @@ -88,11 +89,9 @@ queryStr.append("PREFIX rdf: \n"); queryStr.append("SELECT DISTINCT ?c ?pLabel ?entities\n"); -// if (targetGraphURIs != null) { -// for (String graphURI : targetGraphURIs) { -// queryStr.append("FROM <"); -// queryStr.append(graphURI); -// queryStr.append(">\n"); -// } -// } + if (targetGraphURI != null) { + queryStr.append("FROM <"); + queryStr.append(targetGraphURI); + queryStr.append(">\n"); + } queryStr.append("WHERE{\n"); queryStr.append(" ?cp <").append(URICollection.PROPERTY_VOID_CLASS).append("> ?c. \n"); @@ -133,5 +132,5 @@ try { // long start = System.currentTimeMillis(); - qexec = QueryExecutionFactory.create(query, model); + qexec = QueryExecutionFactory.create(query, dataset.getModel()); results = qexec.execSelect(); // long end = System.currentTimeMillis(); @@ -177,7 +176,9 @@ - public ClassLink[] getNextClass(String[] graphURIs, String originClass, int limit, boolean countLinks) throws Exception{ - String[] targetGraphURIs = filterGraphURIs(graphURIs); - + public ClassLink[] getNextClass(String graphURI, String originClass, int limit, boolean countLinks) throws Exception{ + String targetGraphURI = filterGraphURIs(graphURI); + Dataset dataset = crawledMetadata.getDataset(targetGraphURI); + + StringBuffer queryStr = new StringBuffer(); queryStr.append("PREFIX owl: \n"); @@ -188,11 +189,9 @@ queryStr.append("SELECT DISTINCT ?indPropCat ?c ?dat ?d ?p ?numLnkInsStart ?numLnkInsEnd ?numInsDom ?numInsRan ?numTriples\n"); -// if (targetGraphURIs != null) { -// for (String graphURI : targetGraphURIs) { -// queryStr.append("FROM <"); -// queryStr.append(graphURI); -// queryStr.append(">\n"); -// } -// } + if (targetGraphURI != null) { + queryStr.append("FROM <"); + queryStr.append(targetGraphURI); + queryStr.append(">\n"); + } queryStr.append("WHERE{\n"); @@ -254,5 +253,5 @@ try { long start = System.currentTimeMillis(); - qexec = QueryExecutionFactory.create(query, model); + qexec = QueryExecutionFactory.create(query, dataset.getModel()); results = qexec.execSelect(); long end = System.currentTimeMillis(); @@ -363,9 +362,14 @@ */ - public LabelMap[] getLabels(String[] graphURIs, String[] resourceURIs, + public LabelMap[] getLabels(String graphURI, String[] resourceURIs, String language) throws Exception { - if (resourceURIs == null || resourceURIs.length == 0) { - return new LabelMap[0]; - } +// if (resourceURI == null || resourceURIs.length == 0) { +// return new LabelMap[0]; +// } + + String targetGraphURI = filterGraphURIs(graphURI); + Dataset dataset = crawledMetadata.getDataset(targetGraphURI); + + StringBuffer queryStr = new StringBuffer(); queryStr.append("PREFIX owl: \n"); @@ -373,10 +377,8 @@ queryStr.append("PREFIX rdf: \n"); queryStr.append("SELECT DISTINCT ?res ?label \n"); - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (targetGraphURI != null) { queryStr.append("FROM <"); - queryStr.append(graphURI); + queryStr.append(targetGraphURI); queryStr.append(">\n"); - } } queryStr.append("WHERE{\n"); @@ -399,5 +401,5 @@ Query query = QueryFactory.create(queryStr.toString()); - QueryExecution qexec = QueryExecutionFactory.create(query, model); + QueryExecution qexec = QueryExecutionFactory.create(query, dataset.getModel()); ResultSet results = qexec.execSelect(); Index: SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/EndpointAnalyzer.java =================================================================== --- SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/EndpointAnalyzer.java (revision 268) +++ SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/EndpointAnalyzer.java (revision 269) @@ -148,5 +148,5 @@ */ - public List getOWLClassList(String[] graphURIs, String[] keywords, + public List getOWLClassList(String graphURI, String[] keywords, String language, @@ -157,5 +157,5 @@ // public List - public SClass[] getOWLClasses(String[] graphURIs, String[] keywords, + public SClass[] getOWLClasses(String graphURI, String[] keywords, String language, @@ -170,10 +170,10 @@ queryStr.append("SELECT DISTINCT ?c ?pLabel \n"); } - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { +// for (String graphURI : graphURIs) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } +// } } queryStr.append("WHERE{\n"); @@ -271,5 +271,5 @@ } - public SClass[] listClasses(String[] graphURIs, boolean countInstances) + public SClass[] listClasses(String graphURI, boolean countInstances) throws Exception { StringBuffer queryStr = new StringBuffer(); @@ -283,10 +283,10 @@ } - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { +// for (String graphURI : graphURIs) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } +// } } queryStr.append("WHERE{\n"); @@ -374,5 +374,5 @@ * @since 28.01.2014 */ - public Instance[] getInstances(String[] graphURIs, String keyword) + public Instance[] getInstances(String graphURI, String keyword) throws Exception { StringBuffer queryStr = new StringBuffer(); @@ -381,10 +381,8 @@ queryStr.append("PREFIX rdf: \n"); queryStr.append("SELECT DISTINCT ?ins ?c \n"); - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } } queryStr.append("WHERE{\n"); @@ -450,15 +448,15 @@ * @since 28.01.2014 */ - public ClassLink[] getNextClass(String[] graphURIs, String originClass, + public ClassLink[] getNextClass(String graphURI, String originClass, int limit, boolean countLinks) throws Exception { - ClassLink[] cLinks = getNextClassSub(graphURIs, originClass, limit, + ClassLink[] cLinks = getNextClassSub(graphURI, originClass, limit, false); if (countLinks) { - cLinks = countLinks(graphURIs, originClass, cLinks); + cLinks = countLinks(graphURI, originClass, cLinks); } return cLinks; } - private ClassLink[] getNextClassSub(String[] graphURIs, String originClass, + private ClassLink[] getNextClassSub(String graphURI, String originClass, int limit, boolean countLinks) throws Exception { StringBuffer queryStr = new StringBuffer(); @@ -474,10 +472,8 @@ } - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } } queryStr.append("WHERE{\n"); @@ -615,5 +611,5 @@ * @since 28.01.2014 */ - public ClassLink[] getNextClassViaInstanceLink(String[] graphURIs, + public ClassLink[] getNextClassViaInstanceLink(String graphURI, String originClass, int limit) throws Exception { StringBuffer queryStr = new StringBuffer(); @@ -622,10 +618,10 @@ queryStr.append("PREFIX rdf: \n"); queryStr.append("SELECT DISTINCT ?pf ?pr (COUNT(?pf) AS ?numOfForwardLinks) (COUNT(?pr) AS ?numOfReverseLinks) (COUNT(DISTINCT(?insOrg)) AS ?numOfOriginInstances) (COUNT(DISTINCT(?ins)) AS ?numOfLinkedInstances) ?c \n"); - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { +// for (String graphURI : graphURIs) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } +// } } queryStr.append("WHERE{\n"); @@ -757,5 +753,5 @@ * @since 28.01.2014 */ - public InstanceLink[] getNextInstancesViaInstanceLink(String[] graphURIs, + public InstanceLink[] getNextInstancesViaInstanceLink(String graphURI, String originInstance, int limit) throws Exception { StringBuffer queryStr = new StringBuffer(); @@ -764,10 +760,8 @@ queryStr.append("PREFIX rdf: \n"); queryStr.append("SELECT DISTINCT ?pf ?pr ?ins ?c \n"); - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } } queryStr.append("WHERE{\n"); @@ -833,9 +827,9 @@ } - public LabelMap[] getLabels(String[] graphURIs, String[] resourceURIs, + public LabelMap[] getLabels(String graphURI, String[] resourceURIs, String language) throws Exception { - if (resourceURIs == null || resourceURIs.length == 0) { - return new LabelMap[0]; - } +// if (resourceURIs == null || resourceURIs.length == 0) { +// return new LabelMap[0]; +// } StringBuffer queryStr = new StringBuffer(); queryStr.append("PREFIX owl: \n"); @@ -843,10 +837,8 @@ queryStr.append("PREFIX rdf: \n"); queryStr.append("SELECT DISTINCT ?res ?label \n"); - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } } queryStr.append("WHERE{\n"); @@ -896,5 +888,5 @@ } - public ClassLink[] countLinks(String[] graphURIs, String startClassURI, + public ClassLink[] countLinks(String graphURI, String startClassURI, ClassLink[] classLinks) throws Exception { if (classLinks == null || classLinks.length == 0) { @@ -908,10 +900,8 @@ queryStr.append("SELECT (COUNT(?os) AS ?numOfLinks) (COUNT(DISTINCT(?is)) AS ?numOfOriginInstances) (COUNT(DISTINCT(?os)) AS ?numOfLinkedInstances) \n"); - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } } @@ -1003,10 +993,8 @@ queryStr.append("SELECT (COUNT(?ics) AS ?numOfOriginClassInstances) \n"); - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } } @@ -1056,10 +1044,8 @@ queryStr.append("SELECT (COUNT(?ocs) AS ?numOfLinkedClassInstances) \n"); - if (graphURIs != null) { - for (String graphURI : graphURIs) { + if (graphURI != null) { queryStr.append("FROM <"); queryStr.append(graphURI); queryStr.append(">\n"); - } } Index: SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/RDFSchemaAnalyzerFactory.java =================================================================== --- SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/RDFSchemaAnalyzerFactory.java (revision 267) +++ SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/OWL/RDFSchemaAnalyzerFactory.java (revision 269) @@ -8,5 +8,6 @@ import org.biohackathon.SPARQLBuilder.endpointMetadata.MetadataManager; -import jp.riken.accc.db.rdf.crawler.dataStructure.sparql.JenaModelGenerator; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.JenaModelGenerator; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.crawler.CrawledMetadata; public class RDFSchemaAnalyzerFactory { @@ -47,16 +48,15 @@ } - public MetadataFile[] getMetadataFiles(){ - return metadataManager.getMetadataFiles(); + public CrawledMetadata[] getMetadataFiles(){ + return metadataManager.getCrawlerMetadataList(); } public RDFSchemaAnalyzer create(String uri) throws Exception{ - MetadataFile mFile = metadataManager.getMetadataFile(uri); - if( mFile == null ){ + CrawledMetadata crawledMetadata = metadataManager.getCrawledMetadata(uri); + if( crawledMetadata == null ){ return new EndpointAnalyzer(uri); }else{ - JenaModelGenerator jmGene = new JenaModelGenerator(mFile.getFilePath()); - return new AcquiredStructureAnalyzer(jmGene.getEndpointURI(), jmGene.getGraphURIs(), jmGene.getModel()); + return new AcquiredStructureAnalyzer(crawledMetadata); } } Index: SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/endpointMetadata/MetadataFile.java =================================================================== --- SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/endpointMetadata/MetadataFile.java (revision 267) +++ SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/endpointMetadata/MetadataFile.java (revision 269) @@ -5,5 +5,5 @@ import java.util.Calendar; -import jp.riken.accc.db.rdf.crawler.dataStructure.sparql.URICollection; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.URICollection; import org.apache.jena.riot.RDFDataMgr; Index: SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/endpointMetadata/MetadataManager.java =================================================================== --- SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/endpointMetadata/MetadataManager.java (revision 267) +++ SPARQLBuilderWWW2016/src/java/org/biohackathon/SPARQLBuilder/endpointMetadata/MetadataManager.java (revision 269) @@ -5,7 +5,10 @@ import java.util.Set; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.crawler.CrawledMetadata; +import jp.riken.accc.db.sparqlBuilderMetadata.crawler.dataStructure.sparql.crawler.CrawledMetadataFileReader; + public class MetadataManager { - HashMap metadataTable = null; + HashMap metadataTable = null; /* @@ -22,8 +25,8 @@ }*/ - public MetadataFile[] getMetadataFiles(){ - return metadataTable.values().toArray(new MetadataFile[0]); + public CrawledMetadata[] getCrawlerMetadataList(){ + return metadataTable.values().toArray(new CrawledMetadata[0]); } - public MetadataFile getMetadataFile(String uri){ + public CrawledMetadata getCrawledMetadata(String uri){ return metadataTable.get(uri); } @@ -41,5 +44,5 @@ public void init(String metadataDirStr) throws Exception{ - metadataTable = new HashMap(); + metadataTable = new HashMap(); File metadataDir = new File(metadataDirStr); File[] files = null; @@ -58,17 +61,17 @@ } for(File file: files){ - MetadataFile metadataFile = null; + CrawledMetadata crawledMetadata = null; try{ - metadataFile = new MetadataFile(file); - String uri = metadataFile.getEndpointURI(); + crawledMetadata = CrawledMetadataFileReader.readFile(file.getCanonicalPath()); + String uri = crawledMetadata.getEndpointURI(); if( uri != null ){ if( metadataTable.containsKey(uri)){ - MetadataFile tempMF = metadataTable.get(uri); - if(tempMF.getEndDateTime().before(metadataFile.getEndDateTime()) ){ + CrawledMetadata tempMF = metadataTable.get(uri); + if(tempMF.getDefaultDataset().getCrawlLog().crawlEndTime.before(crawledMetadata.getDefaultDataset().getCrawlLog().crawlEndTime) ){ metadataTable.remove(uri); - metadataTable.put(uri, metadataFile); + metadataTable.put(uri, crawledMetadata); } }else{ - metadataTable.put(uri, metadataFile); + metadataTable.put(uri, crawledMetadata); } }