litvinovg/w2phtml
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 21 Packages Wiki Activity Actions

Modified metadata structure

Browse source
This commit is contained in:
Georgy Litvinov 2020-11-20 16:52:57 +01:00
parent 47bceb3461
commit 8741ed024f
6 changed files with 286 additions and 234 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build.gradle
View file

@ -2,7 +2,7 @@ apply plugin: 'java'
repositories{
mavenCentral()
}
def releaseVersion = "0.8.4"
def releaseVersion = "0.8.6"
def propertyFile = file "src/main/java/w2phtml/project.properties"
Properties properties = new Properties()
propertyFile.withReader { properties.load(it) }

57
src/main/java/pro/litvinovg/w2phtml/gui/ConfigurationWindow.java
View file

@ -274,11 +274,19 @@ public class ConfigurationWindow extends JFrame {
configuration.put("page_break_style", tf_pageBreakInlineStyle);
tf_pageBreakInlineStyle.setText("height:3em;margin-top:1em;margin-bottom:1em;background-color:#f6f6f6;");
tf_pageBreakInlineStyle.setColumns(10);
JCheckBox cb_Pagination = new JCheckBox("Pagination");
configuration.put("pagination", cb_Pagination);
GroupLayout gl_panel_configHTML = new GroupLayout(panel_configHTML);
gl_panel_configHTML.setHorizontalGroup(
gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
.addGroup(gl_panel_configHTML.createSequentialGroup()
.addGap(32)
.addGroup(gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
.addGroup(gl_panel_configHTML.createSequentialGroup()
.addComponent(cb_Pagination, GroupLayout.PREFERRED_SIZE, 303, GroupLayout.PREFERRED_SIZE)
.addContainerGap())
.addGroup(gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
.addGroup(gl_panel_configHTML.createSequentialGroup()
.addGroup(gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
@ -322,7 +330,7 @@ public class ConfigurationWindow extends JFrame {
.addComponent(tf_inputFile, GroupLayout.DEFAULT_SIZE, 616, Short.MAX_VALUE)
.addGap(12)
.addComponent(btnChooseFile, GroupLayout.PREFERRED_SIZE, 113, GroupLayout.PREFERRED_SIZE)))
.addGap(48)))))
.addGap(48))))))
);
gl_panel_configHTML.setVerticalGroup(
gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
@ -381,7 +389,9 @@ public class ConfigurationWindow extends JFrame {
.addComponent(cb_IgnoreEmptyParagraphs)
.addPreferredGap(ComponentPlacement.UNRELATED)
.addComponent(cb_UseMathJax)
.addContainerGap(298, Short.MAX_VALUE))
.addPreferredGap(ComponentPlacement.UNRELATED)
.addComponent(cb_Pagination)
.addContainerGap(271, Short.MAX_VALUE))
);
panel_configHTML.setLayout(gl_panel_configHTML);
return panel_configHTML;
@ -499,11 +509,19 @@ public class ConfigurationWindow extends JFrame {
tf_pageBreakInlineStyle.setColumns(10);
JLabel lb_pageBreakInlineStyle = new JLabel("Page break inline style");
JCheckBox cb_Pagination = new JCheckBox("Pagination");
configuration.put("pagination", cb_Pagination);
GroupLayout gl_panel_configHTML = new GroupLayout(panel_configHTML);
gl_panel_configHTML.setHorizontalGroup(
gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
.addGroup(gl_panel_configHTML.createSequentialGroup()
.addGap(32)
.addGroup(gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
.addGroup(gl_panel_configHTML.createSequentialGroup()
.addComponent(cb_Pagination, GroupLayout.PREFERRED_SIZE, 303, GroupLayout.PREFERRED_SIZE)
.addContainerGap())
.addGroup(gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
.addGroup(gl_panel_configHTML.createSequentialGroup()
.addComponent(lb_pageBreakInlineStyle, GroupLayout.PREFERRED_SIZE, 191, GroupLayout.PREFERRED_SIZE)
@ -556,7 +574,7 @@ public class ConfigurationWindow extends JFrame {
.addComponent(lb_SplitByLevel, GroupLayout.PREFERRED_SIZE, 295, GroupLayout.PREFERRED_SIZE)
.addGap(42)
.addComponent(tf_SplitByLevel, GroupLayout.PREFERRED_SIZE, 175, GroupLayout.PREFERRED_SIZE)))
.addGap(340))))
.addGap(340)))))
);
gl_panel_configHTML.setVerticalGroup(
gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
@ -625,7 +643,9 @@ public class ConfigurationWindow extends JFrame {
.addComponent(cb_SplitWholePagesOnly)
.addPreferredGap(ComponentPlacement.UNRELATED)
.addComponent(cb_UseMathJax)
.addContainerGap(258, Short.MAX_VALUE))
.addPreferredGap(ComponentPlacement.RELATED)
.addComponent(cb_Pagination)
.addContainerGap(235, Short.MAX_VALUE))
);
panel_configHTML.setLayout(gl_panel_configHTML);
return panel_configHTML;
@ -817,11 +837,26 @@ public class ConfigurationWindow extends JFrame {
tf_pageBreakInlineStyle.setText("height:3em;margin-top:1em;margin-bottom:1em;background-color:#f6f6f6;");
tf_pageBreakInlineStyle.setColumns(10);
JCheckBox cb_Pagination = new JCheckBox("Pagination");
configuration.put("pagination", cb_Pagination);
GroupLayout gl_panel_configHTML = new GroupLayout(panel_configHTML);
gl_panel_configHTML.setHorizontalGroup(
gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
gl_panel_configHTML.createParallelGroup(Alignment.TRAILING)
.addGroup(gl_panel_configHTML.createSequentialGroup()
.addGap(23)
.addComponent(lb_pageBreakInlineStyle, GroupLayout.PREFERRED_SIZE, 191, GroupLayout.PREFERRED_SIZE)
.addGap(12)
.addComponent(tf_pageBreakInlineStyle, GroupLayout.DEFAULT_SIZE, 717, Short.MAX_VALUE)
.addGap(21))
.addGroup(Alignment.LEADING, gl_panel_configHTML.createSequentialGroup()
.addGap(32)
.addGroup(gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
.addGroup(gl_panel_configHTML.createSequentialGroup()
.addComponent(cb_Pagination, GroupLayout.PREFERRED_SIZE, 303, GroupLayout.PREFERRED_SIZE)
.addContainerGap())
.addGroup(gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
.addGroup(gl_panel_configHTML.createSequentialGroup()
.addGroup(gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
@ -871,13 +906,7 @@ public class ConfigurationWindow extends JFrame {
.addGroup(gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
.addComponent(btn_metadataChoose, GroupLayout.PREFERRED_SIZE, 113, GroupLayout.PREFERRED_SIZE)
.addComponent(btnChooseFile, GroupLayout.PREFERRED_SIZE, 113, GroupLayout.PREFERRED_SIZE))))
.addGap(70))))
.addGroup(Alignment.TRAILING, gl_panel_configHTML.createSequentialGroup()
.addGap(23)
.addComponent(lb_pageBreakInlineStyle, GroupLayout.PREFERRED_SIZE, 191, GroupLayout.PREFERRED_SIZE)
.addGap(12)
.addComponent(tf_pageBreakInlineStyle, GroupLayout.DEFAULT_SIZE, 717, Short.MAX_VALUE)
.addGap(21))
.addGap(70)))))
);
gl_panel_configHTML.setVerticalGroup(
gl_panel_configHTML.createParallelGroup(Alignment.LEADING)
@ -944,7 +973,9 @@ public class ConfigurationWindow extends JFrame {
.addComponent(cb_IgnoreEmptyParagraphs)
.addPreferredGap(ComponentPlacement.UNRELATED)
.addComponent(cb_UseMathJax)
.addContainerGap(234, Short.MAX_VALUE))
.addPreferredGap(ComponentPlacement.UNRELATED)
.addComponent(cb_Pagination)
.addContainerGap(207, Short.MAX_VALUE))
);
panel_configHTML.setLayout(gl_panel_configHTML);
return panel_configHTML;

56
src/main/java/w2phtml/rdf/DocumentPart.java
View file

@ -1,8 +1,10 @@
package w2phtml.rdf;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import javax.xml.transform.OutputKeys;
@ -35,10 +37,11 @@ public class DocumentPart {
private String parentPath;
private String name = null;
private String order = null;
private HashMap<String,Set<String>> metadata = null;
private Metadata metadata = null;
public DocumentPart(XhtmlDocument document) {
public DocumentPart(XhtmlDocument document,Metadata metadata) {
this.excerptDoc = document;
this.metadata = metadata;
extractPath();
extractName();
extractNumber();
@ -46,8 +49,17 @@ public class DocumentPart {
calculateParentPath();
extractAnnotationMetadata();
}
public DocumentPart(String path, Metadata metadata) {
this.path = path;
this.metadata = metadata;
this.name = "";
extractNumber();
this.body = "";
calculateParentPath();
}
private void extractAnnotationMetadata() {
metadata = new HashMap<String,Set<String>>();
if (excerptDoc == null) {
System.out.println("Error. Excerpt doc is null!");
return;
@ -65,24 +77,12 @@ public class DocumentPart {
if (metaElements.hasAttribute("name") && metaElements.hasAttribute("content")) {
String metaName = metaElements.getAttribute("name");
String metaValue = metaElements.getAttribute("content");
if (!metadata.containsKey(metaName)) {
Set<String> names = new HashSet<String>();
metadata.put(metaName, names);
}
Set<String> names = metadata.get(metaName);
names.add(metaValue);
metadata.addMetadata(order, metaName, metaValue);
}
}
}
}
public DocumentPart(String path) {
metadata = new HashMap<String,Set<String>>();
this.path = path;
this.name = "";
extractNumber();
this.body = "";
calculateParentPath();
}
private void extractName() {
Element excerptContentNode = excerptDoc.getContentNode();
this.name = excerptContentNode.getAttribute("name");
@ -105,7 +105,7 @@ public class DocumentPart {
public void setOrder(String order) {
this.order = order;
}
public String getSequentionalNumber() {
public String getOrder() {
return order;
}
public String getParentPath() {
@ -119,25 +119,15 @@ public class DocumentPart {
}
public String getName() {
if (metadata != null) {
Set<String> nameSet = metadata.get(DC_TITLE);
if (nameSet != null && !nameSet.isEmpty()) {
for (String value : nameSet) {
//return first one
return value;
String nameFromMeta = metadata.getFirstValue(order,DC_TITLE);
if (!nameFromMeta.isEmpty()) {
return nameFromMeta;
}
}
}
if (name.isEmpty()) {
return "NONAME";
}
if (!name.isEmpty()) {
return name;
}
public void setMetadata(HashMap<String,Set<String>> metadata) {
this.metadata = metadata;
}
public HashMap<String,Set<String>> getMetadata(){
return metadata;
return "No name";
}
private void extractPath() {

71
src/main/java/w2phtml/rdf/DocumentStructure.java
View file

@ -4,7 +4,9 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.Vector;
@ -28,6 +30,7 @@ public class DocumentStructure {
private static final String TEXT_EXCERPT = "textExcerpt";
private static final String TOC_LEVEL = "TOCLevel";
private static final String TABLE_OF_CONTENTS = "TOC";
private static final String PARTICIPANT = "publicationParticipant";
private static final String PUBLICATION = "publication";
private static final String BOOK = "book";
private static final String JOURNAL = "journal";
@ -49,11 +52,13 @@ public class DocumentStructure {
private HashMap<String, Resource> tocLevels;
private HashMap<String, DocumentPart> inputParts;
private Metadata metadata;
public DocumentStructure(Vector<XhtmlDocument> files,String fileName, XhtmlConfig config) {
public DocumentStructure(Vector<XhtmlDocument> files,String fileName, XhtmlConfig config,Metadata metadata) {
this.tocLevels = new HashMap<String, Resource>();
this.inputParts = new HashMap<String, DocumentPart>();
this.m = ModelFactory.createOntologyModel();
this.metadata = metadata;
this.textOntology = ModelFactory.createOntologyModel();
this.tocLevelClass = textOntology.createClass(TS + TOC_LEVEL);
this.tocClass = textOntology.createClass(TS + TABLE_OF_CONTENTS);
@ -65,7 +70,7 @@ public class DocumentStructure {
this.excerptClass = textOntology.createClass(TS + excerptType);
for(int i = 0 ; i< files.size();i++) {
XhtmlDocument inputDoc = files.get(i);
DocumentPart part = new DocumentPart(inputDoc);
DocumentPart part = new DocumentPart(inputDoc, metadata);
part.setOrder(Integer.toString(i));
addPart(part);
}
@ -74,7 +79,7 @@ public class DocumentStructure {
private void setPublicationType(String publicationType) {
if (publicationType.equals(documentType)) {
if (publicationType.equals(ELENPH_ARTICLE)) {
this.documentType = ELENPH_ARTICLE;
this.excerptType = ELENPH_EXCERPT;
} else if (publicationType.equals(BOOK)){
@ -99,7 +104,7 @@ public class DocumentStructure {
if (levels[i].equals("0")) {
String emptyPath = createEmptyPath(levels, i);
if (!inputParts.containsKey(emptyPath)) {
DocumentPart emptyPart = new DocumentPart(emptyPath);
DocumentPart emptyPart = new DocumentPart(emptyPath, metadata);
addPart(emptyPart);
}
}
@ -166,28 +171,50 @@ public class DocumentStructure {
private void createDocument(DocumentPart docPart) {
String documentURI = TS + documentType + "/" + PARSERNAME + "_" + docID ;
Resource document = m.createResource(documentURI, documentClass);
document.addProperty( RDFS.label, docPart.getName());
Resource mainResource = m.createResource(documentURI, documentClass);
mainResource.addProperty( RDFS.label, docPart.getName());
String tocURI = TS + TABLE_OF_CONTENTS + "/" + PARSERNAME + "_" + docID ;
Resource toc = m.createResource(tocURI, tocClass);
toc.addProperty( RDFS.label, docPart.getName());
Property hasTOC = m.createProperty(TS + "hasTOC");
document.addProperty(hasTOC, toc);
mainResource.addProperty(hasTOC, toc);
addMetadataProperties(document, docPart.getMetadata());
addMetadataProperties(mainResource, docPart);
addAuthor(mainResource, docPart);
tocLevels.put(docPart.getPath(), toc);
attachExcerpt(docPart, toc);
}
private void addMetadataProperties(Resource resource, HashMap<String, Set<String>> metadata) {
Set<String> names = metadata.keySet();
private void addAuthor(Resource resource, DocumentPart docPart) {
String order = docPart.getOrder();
ArrayList<Map<String, String>> sectionMeta = metadata.getSection(order);
for (Map<String, String> map : sectionMeta) {
Set<String> names = map.keySet();
if (names.contains("author given name")) {
attachAuthor(resource, map, order);
}
}
}
private void attachAuthor(Resource resource, Map<String, String> map, String order) {
//String tocURI = TS + PARTICIPANT + "/" + PARSERNAME + "_" + docID ;
//Resource toc = m.createResource(tocURI, tocClass);
}
private void addMetadataProperties(Resource resource, DocumentPart docPart) {
String order = docPart.getOrder();
ArrayList<Map<String, String>> sectionMeta = metadata.getSection(order);
for (Map<String, String> map : sectionMeta) {
Set<String> names = map.keySet();
for (String name : names) {
Set<String> values = metadata.get(name);
if (values != null) {
for (String value : values) {
addMetadata(resource,name,value);
String value = map.get(name);
if (value != null && !value.isEmpty()) {
addMetadata(resource,name,value.trim());
}
}
}
@ -219,11 +246,9 @@ public class DocumentStructure {
private boolean isDefinedInOntology(Resource resource, String name) {
String nameSpace = resource.getNameSpace();
if (nameSpace.contains(TS + excerptType)) {
if (name.equals("author") ||
name.equals("bibliography") ||
if (name.equals("bibliography") ||
name.equals("keywords") ||
name.equals("works") ||
name.equals("affiliation")
name.equals("works")
) {
return true;
}
@ -231,7 +256,6 @@ public class DocumentStructure {
if (nameSpace.contains(TS + documentType)) {
if (name.equals("doi") ||
name.equals("firstPublication") ||
//name.equals("yearAndMonth") ||
name.equals("year") ||
name.equals("issue")
) {
@ -299,7 +323,9 @@ public class DocumentStructure {
Property hasText = m.createProperty(TS + "hasText");
element.addProperty(hasText, excerpt);
if (!docPart.isMasterPart()) {
addMetadataProperties(excerpt,docPart.getMetadata());
addMetadataProperties(excerpt, docPart);
addAuthor(excerpt, docPart);
}
}
@ -329,10 +355,5 @@ public class DocumentStructure {
}
}
}
public void applyMetadata(Metadata metadata) {
for (DocumentPart part: inputParts.values()) {
metadata.apply(part);
}
}
}

67
src/main/java/w2phtml/rdf/Metadata.java
View file

@ -5,9 +5,12 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.slf4j.Logger;
@ -22,10 +25,10 @@ public class Metadata {
private static final String SECTION = "Section";
private static final Logger logger = LoggerFactory.getLogger(Metadata.class);
private HashMap<String, HashMap<String, Set<String>>> sectionsMetadata;
private HashMap<String, ArrayList<Map<String, String>>> sectionsMetadata;
public Metadata() {
sectionsMetadata = new HashMap<String, HashMap<String,Set<String>>>();
sectionsMetadata = new HashMap<String, ArrayList<Map<String, String>>>();
}
@ -90,46 +93,54 @@ public class Metadata {
}
private void processSectionMetadata(Map<String, String> lineMap, String sectionNum) {
HashMap<String, Set<String>> sectionMetadata = getSectionMetadataContainer(sectionNum);
Set<String> names = lineMap.keySet();
for (String name : names) {
Set<String> values = getValueSet(sectionMetadata,name.trim());
String curValue = lineMap.get(name);
if (!curValue.trim().isEmpty()) {
values.add(curValue.trim());
ArrayList<Map<String, String>> sectionMetadata = getSection(sectionNum);
Iterator<Entry<String, String>> iterator = lineMap.entrySet().iterator();
for (;iterator.hasNext();) {
Entry<String, String> entry = iterator.next();
String key = entry.getKey();
String value = entry.getValue();
if (key == null || value == null || key.trim().isEmpty() || value.trim().isEmpty()) {
iterator.remove();
}
}
sectionMetadata.add(lineMap);
}
private Set<String> getValueSet(HashMap<String, Set<String>> sectionMetadata, String name) {
if (!sectionMetadata.containsKey(name)) {
Set<String> values = new HashSet<String>();
sectionMetadata.put(name, values);
public ArrayList<Map<String, String>> getSection(String sectionNum) {
if (sectionNum.equals("0")) {
sectionNum = "";
}
return sectionMetadata.get(name);
}
private HashMap<String, Set<String>> getSectionMetadataContainer(String sectionNum) {
if (!sectionsMetadata.containsKey(sectionNum)) {
HashMap<String,Set<String>> sectionMetadata = new HashMap<String, Set<String>>();
ArrayList<Map<String, String>> sectionMetadata = new ArrayList<Map<String, String>>();
sectionsMetadata.putIfAbsent(sectionNum, sectionMetadata);
}
return sectionsMetadata.get(sectionNum);
}
public void apply(DocumentPart part) {
String sequenceNumber = part.getSequentionalNumber();
if (sequenceNumber == null) {
return;
public String getFirstValue(String section, String metadataName) {
if (section.equals("0")) {
section = "";
}
//root part doesn't have number in csv
if (sequenceNumber.equals("0")) {
sequenceNumber = "";
if (sectionsMetadata.containsKey(section)) {
ArrayList<Map<String, String>> sectionMetadata = sectionsMetadata.get(section);
for (Map<String, String> lineMap : sectionMetadata) {
String value = lineMap.get(metadataName);
if (value != null && !value.trim().isEmpty()) {
return value;
}
HashMap<String, Set<String>> metadata = sectionsMetadata.get(sequenceNumber);
if (metadata != null) {
part.setMetadata(metadata);
}
}
return "";
}
public void addMetadata(String section, String metaName, String metaValue) {
if (section.equals("0")) {
section = "";
}
ArrayList<Map<String, String>> sectionContainer = getSection(section);
HashMap<String, String> map = new HashMap<String,String>();
map.put(metaName, metaValue);
sectionContainer.add(map);
}
}

3
src/main/java/w2phtml/rdf/RDFDocumentResult.java
View file

@ -23,10 +23,9 @@ public class RDFDocumentResult implements OutputFile {
public RDFDocumentResult(Vector<XhtmlDocument> outFiles, String fileName, XhtmlConfig config) {
this.sFileName = Misc.removeExtension(fileName);
this.config = config;
rdfStructure = new DocumentStructure(outFiles,sFileName,config);
Metadata metadata = new Metadata();
metadata.read(config.getCSVMetadataFile());
rdfStructure.applyMetadata(metadata);
rdfStructure = new DocumentStructure(outFiles,sFileName,config,metadata);
rdfStructure.createTree();
}