From 1fc2e5edbded3e2ae18a5df572da562003e1ef19 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Fri, 8 May 2026 16:16:05 +0100 Subject: [PATCH 01/27] WIP - SIARDDK export and import support --- .../modules/postgresql/PostgreSQLHelper.java | 2 +- .../SIARDDK1007FileIndexFileStrategy.java | 7 +++ .../SIARDDK128FileIndexFileStrategy.java | 7 +++ .../SIARDDKFileIndexFileStrategy.java | 7 ++- .../SIARDDKMetadataExportStrategy.java | 16 +++--- .../siard/out/metadata/SIARDMarshaller.java | 17 ++++++ .../out/metadata/StandardSIARDMarshaller.java | 54 +++++++++++++++++-- .../SIARDDK1007DatabaseExportModule.java | 6 +++ .../SIARDDK128DatabaseExportModule.java | 6 +++ .../out/output/SIARDDK128ExportModule.java | 6 ++- .../output/SIARDDKDatabaseExportModule.java | 25 +++++---- .../siard/out/output/SIARDDKExportModule.java | 16 +++--- 12 files changed, 134 insertions(+), 35 deletions(-) diff --git a/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java b/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java index 7d63862a6..de1449ad1 100644 --- a/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java +++ b/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java @@ -59,7 +59,7 @@ public String escapeSchemaName(String schema) { @Override public String escapeTableName(String table) { - return getStartQuote() + table + getEndQuote(); + return getStartQuote() + StringUtils.strip(table, "\"") + getEndQuote(); } /** diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java index a80f8276f..c6e66d8b0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java @@ -18,6 +18,8 @@ import java.util.List; +import dk.sa.xmlns.diark._1_0.fileindex.ObjectFactory; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +38,11 @@ public SIARDDK1007FileIndexFileStrategy() { super(); } + @Override + JAXBElement createFileIndexTypeRootInstance() { + return new ObjectFactory().createFileIndex(createFileIndexTypeInstance()); + } + @Override FileIndexType createFileIndexTypeInstance() { return new FileIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java index d17924986..723712579 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java @@ -17,6 +17,8 @@ package com.databasepreservation.modules.siard.out.metadata; import com.databasepreservation.modules.siard.bindings.siard_dk_128.FileIndexType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.ObjectFactory; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,11 @@ public SIARDDK128FileIndexFileStrategy() { super(); } + @Override + JAXBElement createFileIndexTypeRootInstance() { + return new ObjectFactory().createFileIndex(createFileIndexTypeInstance()); + } + @Override FileIndexType createFileIndexTypeInstance() { return new FileIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java index 17b8a3ed8..beaf91773 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java @@ -12,6 +12,7 @@ import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,8 +57,8 @@ public Object generateXML(DatabaseStructure dbStructure) throws ModuleException String foNbase = baseContainer.getName(count - 1).toString(); // e.g. // AVID.SA.19000.1 - T fileIndexType = createFileIndexTypeInstance(); - List fList = getF(fileIndexType); + JAXBElement fileIndexType = createFileIndexTypeRootInstance(); + List fList = getF(fileIndexType.getValue()); for (Map.Entry entry : md5sums.entrySet()) { @@ -168,6 +169,8 @@ public byte[] addFile(String path) { return digest; } + abstract JAXBElement createFileIndexTypeRootInstance(); + abstract T createFileIndexTypeInstance(); abstract D createFileIndexTypeFInstance(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java index 6856cc368..7fee0e106 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java @@ -36,13 +36,13 @@ public class SIARDDKMetadataExportStrategy implements MetadataExportStrategy { private static final Logger LOGGER = LoggerFactory.getLogger(SIARDDKMetadataExportStrategy.class); - private SIARDMarshaller siardMarshaller; - private MetadataPathStrategy metadataPathStrategy; - private SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; - private SIARDDKDocIndexFileStrategy SIARDDKDocIndexFileStrategy; - private Map exportModuleArgs; - private LOBsTracker lobsTracker; - private SIARDDKAdapter siarddkAdapter; + protected SIARDMarshaller siardMarshaller; + protected MetadataPathStrategy metadataPathStrategy; + protected SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; + protected SIARDDKDocIndexFileStrategy SIARDDKDocIndexFileStrategy; + protected Map exportModuleArgs; + protected LOBsTracker lobsTracker; + protected SIARDDKAdapter siarddkAdapter; private Reporter reporter; @@ -197,7 +197,7 @@ private void writeSchemaFile(SIARDArchiveContainer container, String indexFile, } } - private void createLocalSharedFolder(SIARDArchiveContainer container) { + protected void createLocalSharedFolder(SIARDArchiveContainer container) { Path containerPath = container.getPath(); Path localShared = Paths.get("Schemas/localShared"); File folder = containerPath.resolve(localShared).toFile(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java index ec54b1d78..d6efe5353 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java @@ -33,4 +33,21 @@ public interface SIARDMarshaller { */ public void marshal(String context, String localeSchemaLocation, String JAXBSchemaLocation, OutputStream writer, Object jaxbElement) throws ModuleException; + + /** + * Generate JAXB Marshaller for writing XML object to the archive. + * + * @param archiveClass + * Siard archive class to give JAXB context + * @param localeSchemaLocation + * The locale location of the XML schema for the metadata file. + * @param JAXBSchemaLocation + * The Marshaller.JAXB_SCHEMA_LOCATION. + * @param writer + * The OutputStream to write to. + * @param jaxbElement + * The JAXB element to marshal. + */ + public void marshal(Class archiveClass, String localeSchemaLocation, String JAXBSchemaLocation, + OutputStream writer, Object jaxbElement) throws ModuleException; } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java index fa1d86f47..1a8dfc108 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java @@ -12,9 +12,6 @@ import java.io.OutputStream; import javax.xml.XMLConstants; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Marshaller; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; @@ -26,6 +23,10 @@ import com.databasepreservation.model.exception.ModuleException; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; + public class StandardSIARDMarshaller implements SIARDMarshaller { private static final String ENCODING = "UTF-8"; @@ -78,4 +79,51 @@ public void marshal(String contextStr, String localeSchemaLocation, String JAXBS throw new ModuleException().withMessage("Error while Marshalling JAXB").withCause(e); } } + + @Override + public void marshal(Class archiveClass, String localeSchemaLocation, String JAXBSchemaLocation, + OutputStream writer, Object jaxbElement) throws ModuleException { + + // Set up JAXB marshaller + + JAXBContext context; + try { + context = JAXBContext.newInstance(archiveClass.getPackage().getName(), archiveClass.getClassLoader()); + } catch (JAXBException e) { + throw new ModuleException().withMessage("Error loading JAXBContent").withCause(e); + } + + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema xsdSchema = null; + try { + InputStream in = this.getClass().getResourceAsStream(localeSchemaLocation); + xsdSchema = schemaFactory.newSchema(new StreamSource(in)); + in.close(); + } catch (SAXException e) { + throw new ModuleException() + .withMessage("XSD file has errors: " + getClass().getResource(localeSchemaLocation).getPath()).withCause(e); + } catch (IOException e) { + throw new ModuleException().withMessage("Could not close InputStream").withCause(e); + } + + Marshaller m; + + try { + + m = context.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + m.setProperty(Marshaller.JAXB_ENCODING, ENCODING); + m.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, JAXBSchemaLocation); + + m.setSchema(xsdSchema); + + m.marshal(jaxbElement, writer); + + } catch (JAXBException e) { + if (e.getCause() instanceof SAXParseException) { + LOGGER.error(e.getCause().getMessage()); + } + throw new ModuleException().withMessage("Error while Marshalling JAXB").withCause(e); + } + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java index d251c3570..e775df706 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java @@ -7,6 +7,7 @@ */ package com.databasepreservation.modules.siard.out.output; +import com.databasepreservation.modules.siard.bindings.siard_dk_1007.SiardDiark; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; /** @@ -23,4 +24,9 @@ public SIARDDK1007DatabaseExportModule(SIARDDKExportModule siarddkExportModule) String getJAXBContext() { return SIARDDKConstants.JAXB_CONTEXT_FILEINDEX; } + + @Override + Class getJAXBContextClass() { + return SiardDiark.class; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java index 334aa2017..ab8b88353 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java @@ -7,6 +7,7 @@ */ package com.databasepreservation.modules.siard.out.output; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.SiardDiark; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; /** @@ -23,4 +24,9 @@ public SIARDDK128DatabaseExportModule(SIARDDKExportModule siarddkExportModule) { String getJAXBContext() { return SIARDDKConstants.JAXB_CONTEXT_FILEINDEX_128; } + + @Override + Class getJAXBContextClass() { + return SiardDiark.class; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java index 0c70f2856..9d1876645 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java @@ -14,15 +14,17 @@ package com.databasepreservation.modules.siard.out.output; +import java.util.Map; + import com.databasepreservation.modules.siard.common.adapters.SIARDDK128Adapter; import com.databasepreservation.modules.siard.common.path.SIARDDK128MetadataPathStrategy; import com.databasepreservation.modules.siard.common.path.SIARDDKMetadataPathStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDK128DocIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDK128FileIndexFileStrategy; +import com.databasepreservation.modules.siard.out.metadata.SIARDDK128MetadataExportStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKDocIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKMetadataExportStrategy; -import java.util.Map; /** * @author António Lindo @@ -51,7 +53,7 @@ SIARDDKMetadataPathStrategy createSIARDDKMetadataPathStrategyInstance() { @Override SIARDDKMetadataExportStrategy createSIARDDKMetadataExportStrategyInstance() { - return new SIARDDKMetadataExportStrategy(this, new SIARDDK128Adapter()); + return new SIARDDK128MetadataExportStrategy(this, new SIARDDK128Adapter()); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index e9a85f2cc..031bceb1a 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -7,16 +7,6 @@ */ package com.databasepreservation.modules.siard.out.output; -import com.databasepreservation.model.exception.ModuleException; -import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; -import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; -import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; -import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -25,6 +15,17 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; +import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; +import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; + /** * @author Andreas Kring * @@ -117,7 +118,7 @@ public void finishDatabase() throws ModuleException { OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(siarddkExportModule.getMainContainer(), path, siarddkExportModule.getWriteStrategy()); - siardMarshaller.marshal(getJAXBContext(), metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), + siardMarshaller.marshal(getJAXBContextClass(), metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/fileIndex.xsd", writer, SIARDDKFileIndexFileStrategy.generateXML(null)); @@ -129,4 +130,6 @@ public void finishDatabase() throws ModuleException { } abstract String getJAXBContext(); + + abstract Class getJAXBContextClass(); } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java index 7fd8d54d0..85c7e2dc6 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java @@ -35,14 +35,14 @@ * */ public abstract class SIARDDKExportModule { - private MetadataExportStrategy metadataExportStrategy; - private SIARDArchiveContainer mainContainer; - private ContentExportStrategy contentExportStrategy; - private WriteStrategy writeStrategy; - private ContentPathExportStrategy contentPathExportStrategy; - private MetadataPathStrategy metadataPathStrategy; - private SIARDMarshaller siardMarshaller; - private LOBsTracker lobsTracker; + protected MetadataExportStrategy metadataExportStrategy; + protected SIARDArchiveContainer mainContainer; + protected ContentExportStrategy contentExportStrategy; + protected WriteStrategy writeStrategy; + protected ContentPathExportStrategy contentPathExportStrategy; + protected MetadataPathStrategy metadataPathStrategy; + protected SIARDMarshaller siardMarshaller; + protected LOBsTracker lobsTracker; private Map exportModuleArgs; private SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; From 3f16e32f5626c06b53f5b58e7d4857cf047da9de Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Fri, 15 May 2026 14:34:24 +0100 Subject: [PATCH 02/27] WIP - some SIARDDK and data type export fixes --- .../databasepreservation/modules/jdbc/out/JDBCExportModule.java | 2 +- .../siard/in/metadata/SIARDDK128MetadataImportStrategy.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java index 9d029d73c..92a7b009e 100644 --- a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java +++ b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java @@ -684,7 +684,7 @@ protected void handleSimpleTypeStringDataCell(String data, PreparedStatement ps, protected void handleSimpleTypeNumericExactDataCell(String data, PreparedStatement ps, int index, Cell cell, ColumnStructure column) throws SQLException { - if (data != null) { + if (data != null && !data.isEmpty()) { BigDecimal bd = new BigDecimal(data); ps.setBigDecimal(index, bd); } else { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java index c02344dd1..25bdeb1fe 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java @@ -323,7 +323,7 @@ private TableStructure createContextDocumentationTable() throws ModuleException virtualTable.setRows(contextDocumentationIndex.getDocument().size()); virtualTable.setColumns(createContextDocumentsTableColumns()); virtualTable.setPrimaryKey(createVirtualPrimaryKey( - SIARDDKConstants.CONTEXT_DOCUMENTATION_VIRTUAL_TABLE_PRIMARY_KEY_NAME, SIARDDKConstants.DID)); + SIARDDKConstants.CONTEXT_DOCUMENTATION_VIRTUAL_TABLE_PRIMARY_KEY_NAME, SIARDDKConstants.DOCUMENT_ID)); return virtualTable; } } catch (FileNotFoundException e) { From 82847e7887c587b10df7bbfd590ff9a136810f58 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Mon, 18 May 2026 09:05:23 +0100 Subject: [PATCH 03/27] Add SIARDDK128MetadataExportStrategy --- .../SIARDDK128MetadataExportStrategy.java | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java new file mode 100644 index 000000000..d2a4429e0 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java @@ -0,0 +1,109 @@ +package com.databasepreservation.modules.siard.out.metadata; + +import java.io.IOException; +import java.io.OutputStream; + +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.model.structure.DatabaseStructure; +import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; +import com.databasepreservation.modules.siard.common.adapters.SIARDDKAdapter; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; +import com.databasepreservation.modules.siard.out.write.WriteStrategy; + +/** + * + * @author Alexandre Flores + */ +public class SIARDDK128MetadataExportStrategy extends SIARDDKMetadataExportStrategy { + + public SIARDDK128MetadataExportStrategy(SIARDDKExportModule siarddkExportModule, SIARDDKAdapter siarddkAdapter) { + super(siarddkExportModule, siarddkAdapter); + } + + @Override + public void writeMetadataXML(DatabaseStructure dbStructure, SIARDArchiveContainer outputContainer, + WriteStrategy writeStrategy) throws ModuleException { + // TO-DO: Refactor this into one method in class that can be used by + // SIARDDKDatabaseExportModule also + + // Generate tableIndex.xml + + try { + IndexFileStrategy tableIndexFileStrategy = new SIARDDKTableIndexFileStrategy(lobsTracker, siarddkAdapter); + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + + siardMarshaller.marshal("com.databasepreservation.modules.siard.bindings.siard_dk_128", + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.TABLE_INDEX), + "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/tableIndex.xsd", writer, + tableIndexFileStrategy.generateXML(dbStructure)); + + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing tableIndex.xml to the archive.").withCause(e); + } + + // Generate archiveIndex.xml + + if (exportModuleArgs.get(SIARDDKConstants.ARCHIVE_INDEX) != null) { + try { + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.ARCHIVE_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + IndexFileStrategy archiveIndexFileStrategy = new CommandLineIndexFileStrategy(SIARDDKConstants.ARCHIVE_INDEX, + exportModuleArgs, writer, metadataPathStrategy); + archiveIndexFileStrategy.generateXML(null); + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing archiveIndex.xml to the archive").withCause(e); + } + } + + // Generate contextDocumentationIndex.xml + + if (exportModuleArgs.get(SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX) != null) { + try { + + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + IndexFileStrategy contextDocumentationIndexFileStrategy = new CommandLineIndexFileStrategy( + SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, exportModuleArgs, writer, metadataPathStrategy); + contextDocumentationIndexFileStrategy.generateXML(null); + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing contextDocumentationIndex.xml to the archive") + .withCause(e); + } + } + + if (lobsTracker.getLOBsCount() > 0) { + try { + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + + siardMarshaller.marshal(SIARDDKConstants.JAXB_CONTEXT_DOCINDEX, + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.DOC_INDEX), + "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/docIndex.xsd", writer, + SIARDDKDocIndexFileStrategy.generateXML(dbStructure)); + + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing docIndex.xml to the archive.").withCause(e); + } + } + + createLocalSharedFolder(outputContainer); + } +} From d1125044a457b58dcec0220b0bd5e6dbe04a6564 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Fri, 22 May 2026 16:03:24 +0100 Subject: [PATCH 04/27] Fix SIARDDK128 docindex marshalling --- .../metadata/SIARDDK1007DocIndexFileStrategy.java | 7 +++++++ .../out/metadata/SIARDDK128DocIndexFileStrategy.java | 12 ++++++++++-- .../out/metadata/SIARDDKDocIndexFileStrategy.java | 9 ++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java index ae10da9ae..04720fbf4 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java @@ -12,6 +12,8 @@ import dk.sa.xmlns.diark._1_0.docindex.DocIndexType; import dk.sa.xmlns.diark._1_0.docindex.DocumentType; +import dk.sa.xmlns.diark._1_0.docindex.ObjectFactory; +import jakarta.xml.bind.JAXBElement; /** * @author António Lindo @@ -23,6 +25,11 @@ public SIARDDK1007DocIndexFileStrategy() { super(); } + @Override + JAXBElement createDocIndexTypeRootInstance() { + return new ObjectFactory().createDocIndex(createDocIndexTypeInstance()); + } + @Override DocIndexType createDocIndexTypeInstance() { return new DocIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java index c57ae786c..259518690 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java @@ -7,11 +7,14 @@ */ package com.databasepreservation.modules.siard.out.metadata; +import java.math.BigInteger; +import java.util.List; + import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocumentType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.ObjectFactory; -import java.math.BigInteger; -import java.util.List; +import jakarta.xml.bind.JAXBElement; /** * @author António Lindo @@ -23,6 +26,11 @@ public SIARDDK128DocIndexFileStrategy() { super(); } + @Override + JAXBElement createDocIndexTypeRootInstance() { + return new ObjectFactory().createDocIndex(createDocIndexTypeInstance()); + } + @Override DocIndexType createDocIndexTypeInstance() { return new DocIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java index 62e393839..c0df07692 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java @@ -12,6 +12,7 @@ import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.structure.DatabaseStructure; +import jakarta.xml.bind.JAXBElement; /** * @author Andreas Kring @@ -19,10 +20,10 @@ */ public abstract class SIARDDKDocIndexFileStrategy implements IndexFileStrategy { - private T docIndex; + private JAXBElement docIndex; public SIARDDKDocIndexFileStrategy() { - docIndex = createDocIndexTypeInstance(); + docIndex = createDocIndexTypeRootInstance(); } /* @@ -70,11 +71,13 @@ public D addDoc(int dID, int pID, int mID, int docCollectionNumber, String oFn, setGmlXsd(doc, gmlXsd); } - getDoc(docIndex).add(doc); + getDoc(docIndex.getValue()).add(doc); return doc; } + abstract JAXBElement createDocIndexTypeRootInstance(); + abstract T createDocIndexTypeInstance(); abstract D createDocumentTypeInstance(); From 628cd401368d2fe3c89258152f51593a184aad79 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Wed, 27 May 2026 09:08:09 +0100 Subject: [PATCH 05/27] Fix JAXB context for DocIndex in SIARD DK 128 --- .../siard/out/metadata/SIARDDK128MetadataExportStrategy.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java index d2a4429e0..4a0b82c5f 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java @@ -5,6 +5,7 @@ import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.structure.DatabaseStructure; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.common.adapters.SIARDDKAdapter; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; @@ -90,8 +91,7 @@ public void writeMetadataXML(DatabaseStructure dbStructure, SIARDArchiveContaine String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX); OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); - siardMarshaller.marshal(SIARDDKConstants.JAXB_CONTEXT_DOCINDEX, - metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.DOC_INDEX), + siardMarshaller.marshal(DocIndexType.class, metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.DOC_INDEX), "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/docIndex.xsd", writer, SIARDDKDocIndexFileStrategy.generateXML(dbStructure)); From 4a96d564ce58bab562156a382abb97008ba1a593 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Tue, 2 Jun 2026 09:29:40 +0100 Subject: [PATCH 06/27] Implement asynchronous LOB conversion with HTTP service and enhance BinaryCell to include mimeType --- .../model/data/BinaryCell.java | 11 + .../content/SIARDDKContentExportStrategy.java | 5 +- .../output/SIARDDKDatabaseExportModule.java | 110 +++++++++- .../services/conversion/ConversionResult.java | 9 + .../conversion/HttpLobConversionService.java | 204 ++++++++++++++++++ .../conversion/JobStatusResponse.java | 7 + .../conversion/JobSubmissionResponse.java | 7 + .../conversion/LobConversionService.java | 8 + .../services/conversion/TempFileTracker.java | 33 +++ 9 files changed, 391 insertions(+), 3 deletions(-) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java index 8b9149f11..63f8fbf3b 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java @@ -28,6 +28,7 @@ public class BinaryCell extends Cell implements InputStreamProvider { private InputStreamProvider inputStreamProvider; private String file; private long length; + private String mimeType; /** * Creates a binary cell. This binary cell will mostly just be a wrapper around @@ -75,6 +76,12 @@ public BinaryCell(String id, InputStreamProvider inputStreamProvider) { this.inputStreamProvider = inputStreamProvider; } + public BinaryCell(String id, InputStreamProvider inputStreamProvider, String mimeType) { + super(id); + this.inputStreamProvider = inputStreamProvider; + this.mimeType = mimeType; + } + /** * Creates a binary cell. This binary cell is a wrapper around a * ProvidesInputStream object (whilst also providing Cell functionality). @@ -128,4 +135,8 @@ public String getFile() { public long getLength() { return length; } + + public String getMimeType() { + return mimeType; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 47163f158..9bb53fe9d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -331,7 +331,7 @@ public Row tableRow(Row row) throws ModuleException { InputStream is = new BufferedInputStream(binaryCell.createInputStream()); // Removed because TIKA was a security vulnerability and this feature was not // needed/not fully implemented (see #341) - String mimeType = "unsupported"; + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; IOUtils.closeQuietly(is); // Archive BLOB - simultaneous writing always supported for @@ -357,7 +357,8 @@ public Row tableRow(Row row) throws ModuleException { // Create new FileIndexFileStrategy // Write the BLOB - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), + writeStrategy); InputStream in = blob.getInputStreamProvider().createInputStream(); IOUtils.copy(in, out); IOUtils.closeQuietly(in); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 031bceb1a..b9398be96 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -9,22 +9,39 @@ import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.LinkedList; +import java.util.List; import java.util.Map; +import java.util.Queue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.common.io.providers.PathInputStreamProvider; +import com.databasepreservation.model.data.BinaryCell; +import com.databasepreservation.model.data.Cell; +import com.databasepreservation.model.data.Row; import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; +import com.databasepreservation.modules.siard.services.conversion.ConversionResult; +import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionService; +import com.databasepreservation.modules.siard.services.conversion.LobConversionService; +import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; /** * @author Andreas Kring @@ -35,6 +52,12 @@ public abstract class SIARDDKDatabaseExportModule extends SIARDExportDefault { private SIARDDKExportModule siarddkExportModule; private static final Logger logger = LoggerFactory.getLogger(SIARDDKDatabaseExportModule.class); + private ExecutorService executorService; + private Queue> pendingRowsQueue; + private TempFileTracker tempFileTracker; + private LobConversionService conversionService; + private static final int MAX_QUEUE_SIZE = 100; + public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { super(siarddkExportModule.getContentExportStrategy(), siarddkExportModule.getMainContainer(), siarddkExportModule.getWriteStrategy(), siarddkExportModule.getMetadataExportStrategy(), null); @@ -46,6 +69,12 @@ public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { public void initDatabase() throws ModuleException { super.initDatabase(); + this.tempFileTracker = new TempFileTracker(); + String apiEndpoint = "http://localhost:8080"; + String targetFormat = "image/tiff"; + this.conversionService = new HttpLobConversionService(apiEndpoint, targetFormat, this.tempFileTracker); + this.executorService = Executors.newVirtualThreadPerTaskExecutor(); + // Get docID info from the command line and add these to the LOBsTracker Path pathToArchive = siarddkExportModule.getMainContainer().getPath(); @@ -83,8 +112,47 @@ public void initDatabase() throws ModuleException { } } + @Override + public void handleDataOpenTable(String tableId) throws ModuleException { + // Prepare the FIFO queue for the new table + this.pendingRowsQueue = new LinkedList<>(); + super.handleDataOpenTable(tableId); + } + + @Override + public void handleDataRow(Row row) throws ModuleException { + // 1. Submit the row conversion to the Virtual Thread + Callable conversionTask = () -> processRow(row); + Future futureRow = executorService.submit(conversionTask); + pendingRowsQueue.add(futureRow); + + logger.debug("Submitted row for asynchronous processing. Current queue size: {}", pendingRowsQueue.size()); + // 2. Backpressure: Wait for the queue to drain if it reaches the limit + while (pendingRowsQueue.size() >= MAX_QUEUE_SIZE) { + logger.debug("Pending rows queue has reached the maximum size of {}. Waiting for the oldest task to complete...", + MAX_QUEUE_SIZE); + drainHeadAndExport(); + } + } + + @Override + public void handleDataCloseTable(String tableId) throws ModuleException { + // Process and export all remaining rows in the queue + while (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { + drainHeadAndExport(); + } + super.handleDataCloseTable(tableId); + } + @Override public void finishDatabase() throws ModuleException { + if (executorService != null && !executorService.isShutdown()) { + executorService.shutdown(); + } + if (tempFileTracker != null) { + tempFileTracker.cleanupAll(); + } + super.finishDatabase(); // Write ContextDocumentation to archive @@ -118,7 +186,8 @@ public void finishDatabase() throws ModuleException { OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(siarddkExportModule.getMainContainer(), path, siarddkExportModule.getWriteStrategy()); - siardMarshaller.marshal(getJAXBContextClass(), metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), + siardMarshaller.marshal(getJAXBContextClass(), + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/fileIndex.xsd", writer, SIARDDKFileIndexFileStrategy.generateXML(null)); @@ -132,4 +201,43 @@ public void finishDatabase() throws ModuleException { abstract String getJAXBContext(); abstract Class getJAXBContextClass(); + + private void drainHeadAndExport() throws ModuleException { + Future oldestFuture = pendingRowsQueue.poll(); + if (oldestFuture != null) { + try { + logger.debug("Waiting for the oldest row conversion task to complete. Remaining queue size after polling: {}", + pendingRowsQueue.size()); + Row processedRow = oldestFuture.get(); + + logger.debug("Oldest row conversion task completed. Exporting row with ID: {}", processedRow.getIndex()); + + super.handleDataRow(processedRow); + } catch (InterruptedException | ExecutionException e) { + oldestFuture.cancel(true); + throw new ModuleException().withMessage("Error processing row conversion task").withCause(e); + } + } + } + + private Row processRow(Row row) throws Exception { + logger.debug("Processing row with ID: {} in thread: {}", row.getIndex(), Thread.currentThread().getName()); + List cells = row.getCells(); + + for (int i = 0; i < cells.size(); i++) { + Cell cell = cells.get(i); + + if (cell instanceof BinaryCell binCell) { + try (InputStream originalStream = binCell.createInputStream()) { + ConversionResult result = conversionService.convertLob(cell.getId(), originalStream); + Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFile()), + "image/tiff"); + cells.set(i, newCell); + } + } + } + row.setCells(cells); + logger.debug("Completed processing row with ID: {}", row.getIndex()); + return row; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java new file mode 100644 index 000000000..4b48a9bc5 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java @@ -0,0 +1,9 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Path; + +/** + * @author Gabriel Barros + */ +public record ConversionResult(Path convertedFile, Path reportFile) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java new file mode 100644 index 000000000..b85cf2d89 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -0,0 +1,204 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.Random; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Handles communication with the external format conversion REST API. Uses + * asynchronous polling and in-memory multipart streaming to process large + * objects efficiently. + */ +public class HttpLobConversionService implements LobConversionService { + + private static final Logger log = LoggerFactory.getLogger(HttpLobConversionService.class); + + private static final int MAX_POLLING_ATTEMPTS = 300; + private static final int MAX_NETWORK_RETRIES = 3; + private static final int BASE_POLLING_INTERVAL_MS = 2000; + + private final HttpClient httpClient; + private final String baseUrl; + private final String targetFormat; + private final ObjectMapper objectMapper; + private final TempFileTracker fileTracker; + private final Random random = new Random(); + + public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTracker fileTracker) { + this.baseUrl = baseUrl; + this.targetFormat = targetFormat; + this.fileTracker = fileTracker; + + this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(30)).build(); + } + + @Override + public ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception { + String jobId = submitJob(cellId, inputStream); + waitForCompletion(cellId, jobId); + return downloadAndExtractResult(cellId, jobId); + } + + /** + * Submits the job using a SequenceInputStream to stream the multipart request + * directly, avoiding intermediate disk writes for the payload. + */ + private String submitJob(String cellId, InputStream inputStream) throws Exception { + String boundary = "DbptkBoundary" + System.currentTimeMillis(); + + String header = "--" + boundary + "\r\n" + "Content-Disposition: form-data; name=\"targetFormat\"\r\n\r\n" + + targetFormat + "\r\n" + "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"file\"; filename=\"lob_" + cellId + ".bin\"\r\n" + + "Content-Type: application/octet-stream\r\n\r\n"; + + String footer = "\r\n--" + boundary + "--\r\n"; + + InputStream headerStream = new ByteArrayInputStream(header.getBytes(StandardCharsets.UTF_8)); + InputStream footerStream = new ByteArrayInputStream(footer.getBytes(StandardCharsets.UTF_8)); + + // Chains the header, actual LOB data, and footer without loading the LOB into + // memory + InputStream multipartStream = new SequenceInputStream( + Collections.enumeration(Arrays.asList(headerStream, inputStream, footerStream))); + + HttpRequest submitRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs")) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(BodyPublishers.ofInputStream(() -> multipartStream)).build(); + + // Not using executeWithRetry here because the InputStream is consumed and + // cannot be trivially reset. + HttpResponse submitResponse = httpClient.send(submitRequest, BodyHandlers.ofString()); + + if (submitResponse.statusCode() >= 400) { + throw new RuntimeException("Failed to submit LOB for cell " + cellId + ": " + submitResponse.body()); + } + + JobSubmissionResponse job = objectMapper.readValue(submitResponse.body(), JobSubmissionResponse.class); + return job.id(); + } + + /** + * Polls the job status API until completion or timeout. Includes jitter to + * prevent thundering herd. + */ + private void waitForCompletion(String cellId, String jobId) throws Exception { + for (int i = 0; i < MAX_POLLING_ATTEMPTS; i++) { + HttpRequest statusRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).GET().build(); + + HttpResponse statusResponse = executeWithRetry(statusRequest, BodyHandlers.ofString(), + MAX_NETWORK_RETRIES); + JobStatusResponse status = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); + + switch (status.status().toUpperCase()) { + case "COMPLETED", "DONE", "SUCCESS" -> { + return; + } + case "FAILED", "ERROR", "EVICTED" -> throw new RuntimeException("Server failed to convert cell: " + cellId); + default -> { + long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); // Jittering + Thread.sleep(sleepTime); + } + } + } + throw new RuntimeException("Timeout after waiting for conversion of cell: " + cellId); + } + + /** + * Downloads the resulting ZIP and extracts its contents. + */ + private ConversionResult downloadAndExtractResult(String cellId, String jobId) throws Exception { + Path tempZipFile = Files.createTempFile("siarddk_conv_" + cellId + "_", ".zip"); + fileTracker.track(tempZipFile); + + HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) + .GET().build(); + + executeWithRetry(downloadRequest, BodyHandlers.ofFile(tempZipFile), MAX_NETWORK_RETRIES); + + return extractZipContents(cellId, tempZipFile); + } + + private ConversionResult extractZipContents(String cellId, Path tempZipFile) throws Exception { + Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); + fileTracker.trackDir(extractionDir); + + Path convertedLob = null; + Path reportFile = null; + + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(tempZipFile.toFile()))) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + Path extractedFilePath = extractionDir.resolve(zipEntry.getName()); + + // Zip Slip vulnerability prevention + if (!extractedFilePath.normalize().startsWith(extractionDir)) { + throw new SecurityException("Corrupted ZIP entry: " + zipEntry.getName()); + } + + if (!zipEntry.isDirectory()) { + Files.copy(zis, extractedFilePath); + + if (zipEntry.getName().toLowerCase().contains("report")) { + reportFile = extractedFilePath; + } else { + convertedLob = extractedFilePath; + } + } + } + } + + if (convertedLob == null || reportFile == null) { + throw new RuntimeException("Downloaded ZIP lacks expected format (LOB + Report) for cell: " + cellId); + } + + return new ConversionResult(convertedLob, reportFile); + } + + /** + * Enforces resilient networking via exponential backoff. Intended exclusively + * for idempotent requests (e.g., GET). + */ + private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, + int maxRetries) throws Exception { + Exception lastException = null; + + for (int attempt = 1; attempt <= maxRetries; attempt++) { + try { + return httpClient.send(request, responseBodyHandler); + } catch (IOException e) { + lastException = e; + log.warn("Attempt {} failed for {}: {}", attempt, request.uri(), e.getMessage()); + + if (attempt == maxRetries) + break; + Thread.sleep((long) Math.pow(2, attempt) * 1000); // Exponential backoff: 2s, 4s, 8s... + } + } + throw new IOException("Exhausted all network retries for URI: " + request.uri(), lastException); + } +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java new file mode 100644 index 000000000..79f8551c3 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java @@ -0,0 +1,7 @@ +package com.databasepreservation.modules.siard.services.conversion; + +/** + * @author Gabriel Barros + */ +public record JobStatusResponse(String status) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java new file mode 100644 index 000000000..2abfaf6dd --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java @@ -0,0 +1,7 @@ +package com.databasepreservation.modules.siard.services.conversion; + +/** + * @author Gabriel Barros + */ +public record JobSubmissionResponse(String id) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java new file mode 100644 index 000000000..038e779b4 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.InputStream; + +public interface LobConversionService { + ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception; +} + diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java new file mode 100644 index 000000000..b138a6ace --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java @@ -0,0 +1,33 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TempFileTracker { + private static final Logger LOGGER = LoggerFactory.getLogger(TempFileTracker.class); + private final Queue trackedFiles = new ConcurrentLinkedQueue<>(); + + public void track(Path path) { + trackedFiles.add(path); + } + + public void trackDir(Path dirPath) { + trackedFiles.add(dirPath); + } + + public void cleanupAll() { + for (Path path : trackedFiles) { + try { + Files.deleteIfExists(path); + } catch (Exception e) { + LOGGER.warn("Unable to delete temporary file/directory: " + path, e); + } + } + trackedFiles.clear(); + } +} \ No newline at end of file From f26062e37f35f4cb719816e7134397f259566f47 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Tue, 2 Jun 2026 16:21:02 +0100 Subject: [PATCH 07/27] Enhance HttpLobConversionService and SIARDDKDatabaseExportModule for improved asynchronous LOB processing and error handling --- .../output/SIARDDKDatabaseExportModule.java | 188 ++++++++++++++---- .../conversion/HttpLobConversionService.java | 69 ++++--- .../services/conversion/TempFileTracker.java | 76 ++++++- 3 files changed, 252 insertions(+), 81 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index b9398be96..ce0fa1e01 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -14,15 +14,18 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Queue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -44,20 +47,34 @@ import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; /** + * Handles database export pipeline matching SIARD-DK compliance, incorporating + * high-throughput asynchronous LOB streaming conversion. + * * @author Andreas Kring * */ public abstract class SIARDDKDatabaseExportModule extends SIARDExportDefault { - private SIARDDKExportModule siarddkExportModule; + private final SIARDDKExportModule siarddkExportModule; private static final Logger logger = LoggerFactory.getLogger(SIARDDKDatabaseExportModule.class); private ExecutorService executorService; - private Queue> pendingRowsQueue; private TempFileTracker tempFileTracker; private LobConversionService conversionService; private static final int MAX_QUEUE_SIZE = 100; + // Resilient Pipeline architecture attributes + private BlockingQueue> pendingRowsQueue; + private ExecutorService writerExecutor; + private Future writerTask; + private final AtomicReference writerError = new AtomicReference<>(); + + /** + * Data context tuple to link rows with their transient extracted disk paths. + */ + private record ProcessedRowContext(Row row, List transientPaths) { + } + public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { super(siarddkExportModule.getContentExportStrategy(), siarddkExportModule.getMainContainer(), siarddkExportModule.getWriteStrategy(), siarddkExportModule.getMetadataExportStrategy(), null); @@ -70,10 +87,11 @@ public void initDatabase() throws ModuleException { super.initDatabase(); this.tempFileTracker = new TempFileTracker(); - String apiEndpoint = "http://localhost:8080"; + String apiEndpoint = "http://localhost:8087"; String targetFormat = "image/tiff"; this.conversionService = new HttpLobConversionService(apiEndpoint, targetFormat, this.tempFileTracker); this.executorService = Executors.newVirtualThreadPerTaskExecutor(); + this.writerExecutor = Executors.newSingleThreadExecutor(); // Dedicated single-thread pipeline consumer // Get docID info from the command line and add these to the LOBsTracker @@ -114,33 +132,22 @@ public void initDatabase() throws ModuleException { @Override public void handleDataOpenTable(String tableId) throws ModuleException { - // Prepare the FIFO queue for the new table - this.pendingRowsQueue = new LinkedList<>(); + logger.debug("Opening table '{}'. Initializing asynchronous pipeline...", tableId); + this.pendingRowsQueue = new LinkedBlockingQueue<>(MAX_QUEUE_SIZE); + this.writerError.set(null); + startAsyncWriter(); super.handleDataOpenTable(tableId); } @Override public void handleDataRow(Row row) throws ModuleException { - // 1. Submit the row conversion to the Virtual Thread - Callable conversionTask = () -> processRow(row); - Future futureRow = executorService.submit(conversionTask); - pendingRowsQueue.add(futureRow); - - logger.debug("Submitted row for asynchronous processing. Current queue size: {}", pendingRowsQueue.size()); - // 2. Backpressure: Wait for the queue to drain if it reaches the limit - while (pendingRowsQueue.size() >= MAX_QUEUE_SIZE) { - logger.debug("Pending rows queue has reached the maximum size of {}. Waiting for the oldest task to complete...", - MAX_QUEUE_SIZE); - drainHeadAndExport(); - } + enqueueRow(row); } @Override public void handleDataCloseTable(String tableId) throws ModuleException { - // Process and export all remaining rows in the queue - while (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { - drainHeadAndExport(); - } + logger.debug("Closing table '{}'. Draining remaining items in the pipeline...", tableId); + stopAsyncWriter(); super.handleDataCloseTable(tableId); } @@ -149,6 +156,9 @@ public void finishDatabase() throws ModuleException { if (executorService != null && !executorService.isShutdown()) { executorService.shutdown(); } + if (writerExecutor != null && !writerExecutor.isShutdown()) { + writerExecutor.shutdown(); + } if (tempFileTracker != null) { tempFileTracker.cleanupAll(); } @@ -195,34 +205,111 @@ public void finishDatabase() throws ModuleException { } catch (IOException e) { throw new ModuleException().withMessage("Error writing fileIndex to the archive.").withCause(e); } - } - abstract String getJAXBContext(); + /** + * Starts the sequential pipeline background consumer thread. + */ + private void startAsyncWriter() { + this.writerTask = writerExecutor.submit(() -> { + try { + while (!Thread.currentThread().isInterrupted()) { + Future future = pendingRowsQueue.take(); // Enforces strict sequential order + ProcessedRowContext context = future.get(); // Awaits specific LOB HTTP processing boundary - abstract Class getJAXBContextClass(); + super.handleDataRow(context.row()); - private void drainHeadAndExport() throws ModuleException { - Future oldestFuture = pendingRowsQueue.poll(); - if (oldestFuture != null) { + // Alleviate disk pressure by wiping extracted structures instantly after XML + // writing + cleanupTransientPaths(context.transientPaths()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Pipeline consumer thread was interrupted and is shutting down."); + } catch (ExecutionException e) { + logger.error("A background conversion task failed critically: {}", e.getCause().getMessage()); + writerError.set(e.getCause()); + } catch (Exception e) { + logger.error("An unexpected error occurred during sequential writing.", e); + writerError.set(e); + } + }); + } + + /** + * Gracefully drains the remaining queue, safely stops the consumer and monitors + * failures. + */ + private void stopAsyncWriter() throws ModuleException { + while (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { + if (writerError.get() != null) { + break; + } try { - logger.debug("Waiting for the oldest row conversion task to complete. Remaining queue size after polling: {}", - pendingRowsQueue.size()); - Row processedRow = oldestFuture.get(); + TimeUnit.MILLISECONDS.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } - logger.debug("Oldest row conversion task completed. Exporting row with ID: {}", processedRow.getIndex()); + if (writerTask != null) { + writerTask.cancel(true); + } - super.handleDataRow(processedRow); - } catch (InterruptedException | ExecutionException e) { - oldestFuture.cancel(true); - throw new ModuleException().withMessage("Error processing row conversion task").withCause(e); + // Purge and cancel any remaining futures to avoid thread and resource leakage + if (pendingRowsQueue != null) { + for (Future future : pendingRowsQueue) { + future.cancel(true); + try { + if (future.isDone() && !future.isCancelled()) { + cleanupTransientPaths(future.get().transientPaths()); + } + } catch (Exception ignored) { + } } + pendingRowsQueue.clear(); + } + + if (writerError.get() != null) { + throw new ModuleException().withMessage("Row writing pipeline aborted").withCause(writerError.get()); + } + } + + /** + * Pushes rows into the pipeline, throwing fast if the writer task fails, and + * using non-deadlocking backpressure. + */ + private void enqueueRow(Row row) throws ModuleException { + if (writerError.get() != null) { + throw new ModuleException().withMessage("Pipeline execution halted due to previous background failure") + .withCause(writerError.get()); + } + + Callable conversionTask = () -> processRowAsync(row); + Future future = executorService.submit(conversionTask); + + try { + // Prevents deadlocks if the consumer thread crashes while queue is maxed out + while (!pendingRowsQueue.offer(future, 500, TimeUnit.MILLISECONDS)) { + if (writerError.get() != null) { + future.cancel(true); + throw new ModuleException().withMessage("Pipeline halted while enqueuing row").withCause(writerError.get()); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + future.cancel(true); + throw new ModuleException().withMessage("Row enqueuing process was interrupted").withCause(e); } } - private Row processRow(Row row) throws Exception { - logger.debug("Processing row with ID: {} in thread: {}", row.getIndex(), Thread.currentThread().getName()); + /** + * Processes row columns concurrently on Virtual Threads mapping extracted files + * for lifecycle control. + */ + private ProcessedRowContext processRowAsync(Row row) throws Exception { List cells = row.getCells(); + List transientPaths = new ArrayList<>(); for (int i = 0; i < cells.size(); i++) { Cell cell = cells.get(i); @@ -233,11 +320,28 @@ private Row processRow(Row row) throws Exception { Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFile()), "image/tiff"); cells.set(i, newCell); + + // Track extracted parts to clean them individually later + transientPaths.add(result.convertedFile()); + transientPaths.add(result.reportFile()); + transientPaths.add(result.convertedFile().getParent()); // directory container } } } row.setCells(cells); - logger.debug("Completed processing row with ID: {}", row.getIndex()); - return row; + return new ProcessedRowContext(row, transientPaths); + } + + private void cleanupTransientPaths(List paths) { + if (paths == null || tempFileTracker == null) { + return; + } + for (Path path : paths) { + tempFileTracker.deleteEarly(path); + } } -} + + abstract String getJAXBContext(); + + abstract Class getJAXBContextClass(); +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index b85cf2d89..4f143fb87 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -31,14 +31,16 @@ * Handles communication with the external format conversion REST API. Uses * asynchronous polling and in-memory multipart streaming to process large * objects efficiently. + * + * @author Gabriel Barros */ public class HttpLobConversionService implements LobConversionService { private static final Logger log = LoggerFactory.getLogger(HttpLobConversionService.class); - private static final int MAX_POLLING_ATTEMPTS = 300; private static final int MAX_NETWORK_RETRIES = 3; private static final int BASE_POLLING_INTERVAL_MS = 2000; + private static final int MAX_POLLING_ATTEMPTS = 600; // ~20 minutes maximum wait per file before assuming Zombie Job private final HttpClient httpClient; private final String baseUrl; @@ -51,23 +53,18 @@ public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTra this.baseUrl = baseUrl; this.targetFormat = targetFormat; this.fileTracker = fileTracker; - this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(30)).build(); + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(60)).build(); } @Override public ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception { + log.debug("Initiating conversion pipeline for cell: {}", cellId); String jobId = submitJob(cellId, inputStream); waitForCompletion(cellId, jobId); return downloadAndExtractResult(cellId, jobId); } - /** - * Submits the job using a SequenceInputStream to stream the multipart request - * directly, avoiding intermediate disk writes for the payload. - */ private String submitJob(String cellId, InputStream inputStream) throws Exception { String boundary = "DbptkBoundary" + System.currentTimeMillis(); @@ -81,8 +78,6 @@ private String submitJob(String cellId, InputStream inputStream) throws Exceptio InputStream headerStream = new ByteArrayInputStream(header.getBytes(StandardCharsets.UTF_8)); InputStream footerStream = new ByteArrayInputStream(footer.getBytes(StandardCharsets.UTF_8)); - // Chains the header, actual LOB data, and footer without loading the LOB into - // memory InputStream multipartStream = new SequenceInputStream( Collections.enumeration(Arrays.asList(headerStream, inputStream, footerStream))); @@ -90,57 +85,72 @@ private String submitJob(String cellId, InputStream inputStream) throws Exceptio .header("Content-Type", "multipart/form-data; boundary=" + boundary) .POST(BodyPublishers.ofInputStream(() -> multipartStream)).build(); - // Not using executeWithRetry here because the InputStream is consumed and - // cannot be trivially reset. HttpResponse submitResponse = httpClient.send(submitRequest, BodyHandlers.ofString()); if (submitResponse.statusCode() >= 400) { - throw new RuntimeException("Failed to submit LOB for cell " + cellId + ": " + submitResponse.body()); + log.error("API rejected LOB submission for cell {}. Status: {}, Body: {}", cellId, submitResponse.statusCode(), + submitResponse.body()); + throw new RuntimeException("Failed to submit LOB for cell " + cellId); } JobSubmissionResponse job = objectMapper.readValue(submitResponse.body(), JobSubmissionResponse.class); + log.debug("Successfully dispatched cell {}. Assigned Job ID: {}", cellId, job.id()); return job.id(); } - /** - * Polls the job status API until completion or timeout. Includes jitter to - * prevent thundering herd. - */ private void waitForCompletion(String cellId, String jobId) throws Exception { - for (int i = 0; i < MAX_POLLING_ATTEMPTS; i++) { - HttpRequest statusRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).GET().build(); + log.debug("Awaiting completion of Job {} (Cell {})", jobId, cellId); + for (int attempts = 1; attempts <= MAX_POLLING_ATTEMPTS; attempts++) { + HttpRequest statusRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).GET().build(); HttpResponse statusResponse = executeWithRetry(statusRequest, BodyHandlers.ofString(), MAX_NETWORK_RETRIES); + JobStatusResponse status = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); switch (status.status().toUpperCase()) { case "COMPLETED", "DONE", "SUCCESS" -> { + log.debug("Job {} (Cell {}) completed successfully after {} attempts.", jobId, cellId, attempts); return; } - case "FAILED", "ERROR", "EVICTED" -> throw new RuntimeException("Server failed to convert cell: " + cellId); + case "FAILED", "ERROR", "EVICTED" -> { + log.error("API reported terminal failure for Job {} (Cell {})", jobId, cellId); + throw new RuntimeException("Server failed to convert cell: " + cellId); + } default -> { - long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); // Jittering + if (attempts % 30 == 0) { + log.warn("Job {} (Cell {}) is taking unusually long. Current status: {}. Attempt: {}/{}", jobId, cellId, + status.status(), attempts, MAX_POLLING_ATTEMPTS); + } + long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); Thread.sleep(sleepTime); } } } + log.error("Zombie Job detected. API failed to resolve Job {} (Cell {}) within the maximum polling threshold.", + jobId, cellId); throw new RuntimeException("Timeout after waiting for conversion of cell: " + cellId); } /** - * Downloads the resulting ZIP and extracts its contents. + * Downloads the resulting ZIP and extracts its contents, freeing the ZIP file + * immediately after. */ private ConversionResult downloadAndExtractResult(String cellId, String jobId) throws Exception { Path tempZipFile = Files.createTempFile("siarddk_conv_" + cellId + "_", ".zip"); fileTracker.track(tempZipFile); - HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) - .GET().build(); + try { + HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) + .GET().build(); - executeWithRetry(downloadRequest, BodyHandlers.ofFile(tempZipFile), MAX_NETWORK_RETRIES); + executeWithRetry(downloadRequest, BodyHandlers.ofFile(tempZipFile), MAX_NETWORK_RETRIES); - return extractZipContents(cellId, tempZipFile); + return extractZipContents(cellId, tempZipFile); + } finally { + // Free disk space immediately after extraction + fileTracker.deleteEarly(tempZipFile); + } } private ConversionResult extractZipContents(String cellId, Path tempZipFile) throws Exception { @@ -155,7 +165,6 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr while ((zipEntry = zis.getNextEntry()) != null) { Path extractedFilePath = extractionDir.resolve(zipEntry.getName()); - // Zip Slip vulnerability prevention if (!extractedFilePath.normalize().startsWith(extractionDir)) { throw new SecurityException("Corrupted ZIP entry: " + zipEntry.getName()); } @@ -179,10 +188,6 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr return new ConversionResult(convertedLob, reportFile); } - /** - * Enforces resilient networking via exponential backoff. Intended exclusively - * for idempotent requests (e.g., GET). - */ private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, int maxRetries) throws Exception { Exception lastException = null; @@ -196,7 +201,7 @@ private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.B if (attempt == maxRetries) break; - Thread.sleep((long) Math.pow(2, attempt) * 1000); // Exponential backoff: 2s, 4s, 8s... + Thread.sleep((long) Math.pow(2, attempt) * 1000); } } throw new IOException("Exhausted all network retries for URI: " + request.uri(), lastException); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java index b138a6ace..ae82d21e0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java @@ -1,33 +1,95 @@ package com.databasepreservation.modules.siard.services.conversion; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Comparator; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Tracks and manages the lifecycle of temporary files and directories created + * during the LOB conversion process. + * + * @author Gabriel Barros + */ public class TempFileTracker { private static final Logger LOGGER = LoggerFactory.getLogger(TempFileTracker.class); - private final Queue trackedFiles = new ConcurrentLinkedQueue<>(); + private final Queue trackedPaths = new ConcurrentLinkedQueue<>(); + /** + * Registers a file or directory path to be tracked for final cleanup. * @param + * path The path to track. + */ public void track(Path path) { - trackedFiles.add(path); + if (path != null) { + trackedPaths.add(path); + } } + /** + * Registers a directory path to be tracked for final cleanup. * @param dirPath + * The directory path to track. + */ public void trackDir(Path dirPath) { - trackedFiles.add(dirPath); + track(dirPath); + } + + /** + * Deletes a tracked path immediately (including non-empty directories) and + * removes it from the tracking queue to free resources early. * @param path The + * path to delete immediately. + */ + public void deleteEarly(Path path) { + if (path == null) { + return; + } + try { + deleteRecursively(path); + trackedPaths.remove(path); + } catch (Exception e) { + LOGGER.warn("Unable to delete temporary path early: " + path, e); + } } + /** + * Deletes all remaining tracked files and directories comprehensively, ensuring + * recursive cleanup of nested content. + */ public void cleanupAll() { - for (Path path : trackedFiles) { + for (Path path : trackedPaths) { try { - Files.deleteIfExists(path); + deleteRecursively(path); } catch (Exception e) { - LOGGER.warn("Unable to delete temporary file/directory: " + path, e); + LOGGER.warn("Unable to delete temporary file/directory during bulk cleanup: " + path, e); + } + } + trackedPaths.clear(); + } + + /** + * Helper method to perform safe recursive deletion of paths and directories. + */ + private void deleteRecursively(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + if (Files.isDirectory(path)) { + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException e) { + LOGGER.warn("Failed to delete nested path: " + p, e); + } + }); } + } else { + Files.deleteIfExists(path); } - trackedFiles.clear(); } } \ No newline at end of file From daae1569f32a2db93f52d4372454721195882bd0 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Wed, 3 Jun 2026 16:08:40 +0100 Subject: [PATCH 08/27] Enhance SIARDDKDatabaseExportModule with improved logging and graceful shutdown for asynchronous writer --- .../output/SIARDDKDatabaseExportModule.java | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index ce0fa1e01..1ae908684 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -214,14 +214,24 @@ private void startAsyncWriter() { this.writerTask = writerExecutor.submit(() -> { try { while (!Thread.currentThread().isInterrupted()) { + logger.debug("Consumer thread is waiting to take the oldest row from the queue..."); Future future = pendingRowsQueue.take(); // Enforces strict sequential order + + logger.debug("Oldest row taken. Awaiting its Virtual Thread completion (LOB HTTP boundary)..."); ProcessedRowContext context = future.get(); // Awaits specific LOB HTTP processing boundary + if (context == null) { + logger.info("<< DEQUEUED: Poison Pill received. Safely shutting down the consumer thread."); + break; + } + super.handleDataRow(context.row()); // Alleviate disk pressure by wiping extracted structures instantly after XML // writing cleanupTransientPaths(context.transientPaths()); + logger.debug("<< DEQUEUED: Row [{}] removed from queue and successfully written. Current size: {}/{}", + context.row().getIndex(), pendingRowsQueue.size(), MAX_QUEUE_SIZE); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -241,31 +251,31 @@ private void startAsyncWriter() { * failures. */ private void stopAsyncWriter() throws ModuleException { - while (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { - if (writerError.get() != null) { - break; - } + if (writerError.get() == null) { try { - TimeUnit.MILLISECONDS.sleep(20); - } catch (InterruptedException e) { + logger.debug("|| TABLE END: Injecting Poison Pill into the queue and waiting for consumer to finish..."); + + Future poisonPill = executorService.submit(() -> null); + + while (!pendingRowsQueue.offer(poisonPill, 500, TimeUnit.MILLISECONDS)) { + if (writerError.get() != null) + break; + } + + if (writerTask != null) { + writerTask.get(); + } + + } catch (InterruptedException | ExecutionException e) { Thread.currentThread().interrupt(); + throw new ModuleException().withMessage("Failed to cleanly stop the background writer").withCause(e); } } - if (writerTask != null) { - writerTask.cancel(true); - } - - // Purge and cancel any remaining futures to avoid thread and resource leakage - if (pendingRowsQueue != null) { + // Purge any remaining futures in case of a catastrophic error + if (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { for (Future future : pendingRowsQueue) { future.cancel(true); - try { - if (future.isDone() && !future.isCancelled()) { - cleanupTransientPaths(future.get().transientPaths()); - } - } catch (Exception ignored) { - } } pendingRowsQueue.clear(); } @@ -290,12 +300,21 @@ private void enqueueRow(Row row) throws ModuleException { try { // Prevents deadlocks if the consumer thread crashes while queue is maxed out + boolean waitingLogged = false; while (!pendingRowsQueue.offer(future, 500, TimeUnit.MILLISECONDS)) { if (writerError.get() != null) { future.cancel(true); throw new ModuleException().withMessage("Pipeline halted while enqueuing row").withCause(writerError.get()); } + + if (!waitingLogged) { + logger.debug("|| PAUSED: Queue is full. Suspended database reading. Waiting for space to enqueue row [{}]...", + row.getIndex()); + waitingLogged = true; + } } + logger.debug(">> ENQUEUED: Row [{}] entered the queue. Current size: {}/{}", row.getIndex(), + pendingRowsQueue.size(), MAX_QUEUE_SIZE); } catch (InterruptedException e) { Thread.currentThread().interrupt(); future.cancel(true); @@ -329,6 +348,11 @@ private ProcessedRowContext processRowAsync(Row row) throws Exception { } } row.setCells(cells); + + long readyCount = pendingRowsQueue.stream().filter(Future::isDone).count(); + logger.debug("== READY: Row [{}] finished conversion. Currently {} ready rows waiting in queue.", row.getIndex(), + readyCount + 1); + return new ProcessedRowContext(row, transientPaths); } From 96c8199a0f7652ab1d9c49276f5956e5abd1ff0f Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Tue, 9 Jun 2026 15:30:42 +0100 Subject: [PATCH 09/27] Refactor LOB conversion service to support multiple converted files and enhance job status handling --- .../modules/siard/SIARDDKModuleFactory.java | 78 ++++++++++-- .../output/SIARDDKDatabaseExportModule.java | 75 ++++++++---- .../conversion/HttpLobConversionService.java | 111 ++++++++++++------ .../conversion/LobConversionService.java | 3 +- .../{ => model}/ConversionResult.java | 5 +- .../services/conversion/model/JobStatus.java | 8 ++ .../{ => model}/JobStatusResponse.java | 4 +- .../{ => model}/JobSubmissionResponse.java | 2 +- 8 files changed, 212 insertions(+), 74 deletions(-) rename dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/{ => model}/ConversionResult.java (57%) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java rename dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/{ => model}/JobStatusResponse.java (65%) rename dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/{ => model}/JobSubmissionResponse.java (94%) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java index f1bc0850a..f49f72f93 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java @@ -42,6 +42,9 @@ public abstract class SIARDDKModuleFactory implements DatabaseModuleFactory { public static final String PARAMETER_AS_SCHEMA = "as-schema"; public static final String PARAMETER_LOBS_PER_FOLDER = "lobs-per-folder"; public static final String PARAMETER_LOBS_FOLDER_SIZE = "lobs-folder-size"; + public static final String PARAMETER_LOB_CONVERSION_ENABLED = "lob-conversion"; + public static final String PARAMETER_LOB_CONVERSION_ENDPOINT = "lob-conversion-endpoint"; + public static final String PARAMETER_LOB_CONVERSION_TARGET_FORMAT = "lob-conversion-target-format"; // TODO: As things are now, are we not always generating the '.1' version of // the archive (indicating that the last .[1-9][0-9] should perhaps not be @@ -75,6 +78,20 @@ public abstract class SIARDDKModuleFactory implements DatabaseModuleFactory { .description("The maximum size (in megabytes) of the docCollection folders (default is 1000 MB").required(false) .hasArgument(true).setOptionalArgument(false).valueIfNotSet("1000"); + private static final Parameter lobConversionEnabled = new Parameter().shortName("lc") + .longName(PARAMETER_LOB_CONVERSION_ENABLED).description("Enables asynchronous LOB conversion via HTTP.") + .hasArgument(false).setOptionalArgument(false).required(false).valueIfSet("true").valueIfNotSet("false"); + + private static final Parameter lobConversionEndpoint = new Parameter().shortName("lce") + .longName(PARAMETER_LOB_CONVERSION_ENDPOINT) + .description("The API endpoint URL for the LOB conversion service (default is http://localhost:8087).") + .hasArgument(true).setOptionalArgument(false).required(false).valueIfNotSet("http://localhost:8087"); + + private static final Parameter lobConversionTargetFormat = new Parameter().shortName("lcf") + .longName(PARAMETER_LOB_CONVERSION_TARGET_FORMAT) + .description("Target MIME type format for the LOB conversion (default is image/tiff).").hasArgument(true) + .setOptionalArgument(false).required(false).valueIfNotSet("image/tiff"); + // This is not used now, but will be used later // private static final Parameter clobType = new // Parameter().shortName("ct").longName("clobtype") @@ -120,6 +137,9 @@ public Map getAllParameters() { parameterMap.put(importAsSchema.longName(), importAsSchema); parameterMap.put(lobsPerFolder.longName(), lobsPerFolder); parameterMap.put(lobsFolderSize.longName(), lobsFolderSize); + parameterMap.put(lobConversionEnabled.longName(), lobConversionEnabled); + parameterMap.put(lobConversionEndpoint.longName(), lobConversionEndpoint); + parameterMap.put(lobConversionTargetFormat.longName(), lobConversionTargetFormat); // to be used later... // parameterMap.put(clobType.longName(), clobType); // parameterMap.put(clobLength.longName(), clobLength); @@ -145,18 +165,22 @@ public Parameters getExportModuleParameters() throws UnsupportedModuleException // contextDocumentationIndex, contextDocmentationFolder, // clobType, clobLength), null); - return new Parameters( - Arrays.asList( - folder.inputType(Parameter.INPUT_TYPE.FOLDER).exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - archiveIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN).fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - contextDocumentationIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN) - .fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - contextDocumentationFolder.inputType(Parameter.INPUT_TYPE.FOLDER) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), - lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS)), + return new Parameters(Arrays.asList( + folder.inputType(Parameter.INPUT_TYPE.FOLDER).exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + archiveIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN).fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + contextDocumentationIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN) + .fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + contextDocumentationFolder.inputType(Parameter.INPUT_TYPE.FOLDER) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionEnabled.inputType(Parameter.INPUT_TYPE.COMBOBOX).possibleValues("true", "false") + .defaultSelectedIndex(1).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionEndpoint.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionTargetFormat.inputType(Parameter.INPUT_TYPE.TEXT) + .exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS)), null); } @@ -193,6 +217,21 @@ public DatabaseFilterModule buildExportModule(Map parameters, pLobsFolderSize = parameters.get(lobsFolderSize); } + String pLobConversionEnabled = lobConversionEnabled.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionEnabled))) { + pLobConversionEnabled = parameters.get(lobConversionEnabled); + } + + String pLobConversionEndpoint = lobConversionEndpoint.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionEndpoint))) { + pLobConversionEndpoint = parameters.get(lobConversionEndpoint); + } + + String pLobConversionTargetFormat = lobConversionTargetFormat.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionTargetFormat))) { + pLobConversionTargetFormat = parameters.get(lobConversionTargetFormat); + } + // to be used later... // String pClobType = parameters.get(clobType); // String pClobLength = parameters.get(clobLength); @@ -205,6 +244,9 @@ public DatabaseFilterModule buildExportModule(Map parameters, exportModuleArgs.put(contextDocumentationFolder.longName(), pContextDocumentationFolder); exportModuleArgs.put(lobsPerFolder.longName(), pLobsPerFolder); exportModuleArgs.put(lobsFolderSize.longName(), pLobsFolderSize); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_ENABLED, pLobConversionEnabled); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_ENDPOINT, pLobConversionEndpoint); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_TARGET_FORMAT, pLobConversionTargetFormat); // to be used later... // exportModuleArgs.put(clobType.longName(), pClobType); @@ -227,6 +269,18 @@ public DatabaseFilterModule buildExportModule(Map parameters, exportModuleParameters.add(lobsFolderSize.longName()); exportModuleParameters.add(pLobsFolderSize); } + if (!pLobConversionEnabled.equals(lobConversionEnabled.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_ENABLED); + exportModuleParameters.add(pLobConversionEnabled); + } + if (!pLobConversionEndpoint.equals(lobConversionEndpoint.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_ENDPOINT); + exportModuleParameters.add(pLobConversionEndpoint); + } + if (!pLobConversionTargetFormat.equals(lobConversionTargetFormat.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_TARGET_FORMAT); + exportModuleParameters.add(pLobConversionTargetFormat); + } reporter.exportModuleParameters(getModuleName(), exportModuleParameters.toArray(new String[0])); return createSIARDDKExportModuleInstance(exportModuleArgs).getDatabaseExportModule(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 1ae908684..08461e8a7 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -36,15 +36,17 @@ import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.Row; import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.SIARDDKModuleFactory; import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; -import com.databasepreservation.modules.siard.services.conversion.ConversionResult; import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionService; import com.databasepreservation.modules.siard.services.conversion.LobConversionService; import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; +import com.databasepreservation.utils.ConfigUtils; /** * Handles database export pipeline matching SIARD-DK compliance, incorporating @@ -61,7 +63,8 @@ public abstract class SIARDDKDatabaseExportModule extends SIARDExportDefault { private ExecutorService executorService; private TempFileTracker tempFileTracker; private LobConversionService conversionService; - private static final int MAX_QUEUE_SIZE = 100; + private String targetLobFormat; + private static final Integer MAX_QUEUE_SIZE = ConfigUtils.getProperty(100, "dbptk.siarddk.export.maxQueueSize"); // Resilient Pipeline architecture attributes private BlockingQueue> pendingRowsQueue; @@ -86,10 +89,26 @@ public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { public void initDatabase() throws ModuleException { super.initDatabase(); - this.tempFileTracker = new TempFileTracker(); - String apiEndpoint = "http://localhost:8087"; - String targetFormat = "image/tiff"; - this.conversionService = new HttpLobConversionService(apiEndpoint, targetFormat, this.tempFileTracker); + Map exportModuleArgs = siarddkExportModule.getExportModuleArgs(); + boolean isLobConversionEnabled = Boolean + .parseBoolean(exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_ENABLED, "false")); + + if (isLobConversionEnabled) { + this.tempFileTracker = new TempFileTracker(); + String apiEndpoint = exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_ENDPOINT, + "http://localhost:8087"); + this.targetLobFormat = exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_TARGET_FORMAT, + "image/tiff"); + + this.conversionService = new HttpLobConversionService(apiEndpoint, this.targetLobFormat, this.tempFileTracker); + logger.info("LOB conversion service enabled. Endpoint: '{}', Target Format: '{}'", apiEndpoint, + this.targetLobFormat); + } else { + this.conversionService = null; + this.tempFileTracker = null; + logger.info("LOB conversion service is disabled."); + } + this.executorService = Executors.newVirtualThreadPerTaskExecutor(); this.writerExecutor = Executors.newSingleThreadExecutor(); // Dedicated single-thread pipeline consumer @@ -221,7 +240,7 @@ private void startAsyncWriter() { ProcessedRowContext context = future.get(); // Awaits specific LOB HTTP processing boundary if (context == null) { - logger.info("<< DEQUEUED: Poison Pill received. Safely shutting down the consumer thread."); + logger.debug("<< DEQUEUED: Poison Pill received. Safely shutting down the consumer thread."); break; } @@ -326,24 +345,36 @@ private void enqueueRow(Row row) throws ModuleException { * Processes row columns concurrently on Virtual Threads mapping extracted files * for lifecycle control. */ - private ProcessedRowContext processRowAsync(Row row) throws Exception { + private ProcessedRowContext processRowAsync(Row row) throws ModuleException { List cells = row.getCells(); List transientPaths = new ArrayList<>(); - for (int i = 0; i < cells.size(); i++) { - Cell cell = cells.get(i); - - if (cell instanceof BinaryCell binCell) { - try (InputStream originalStream = binCell.createInputStream()) { - ConversionResult result = conversionService.convertLob(cell.getId(), originalStream); - Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFile()), - "image/tiff"); - cells.set(i, newCell); - - // Track extracted parts to clean them individually later - transientPaths.add(result.convertedFile()); - transientPaths.add(result.reportFile()); - transientPaths.add(result.convertedFile().getParent()); // directory container + if (this.conversionService != null) { + for (int i = 0; i < cells.size(); i++) { + Cell cell = cells.get(i); + + if (cell instanceof BinaryCell binCell) { + try (InputStream originalStream = binCell.createInputStream()) { + ConversionResult result = conversionService.convertLob(cell.getId(), originalStream); + + // TODO: Handle multiple files per cell if needed. Currently assumes single file + // output. + Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFiles().getFirst()), + this.targetLobFormat); + cells.set(i, newCell); + + // Track extracted parts to clean them individually later + transientPaths.addAll(result.convertedFiles()); + transientPaths.add(result.reportFile()); + transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container + } catch (Exception e) { + String errorMsg = String.format( + "Conversion failed for cell '%s' in row %d. " + + "Please check if the LOB service is running and accessible. Detail: %s", + cell.getId(), row.getIndex(), e.getMessage()); + logger.error(errorMsg); + throw new ModuleException().withMessage(errorMsg).withCause(e); + } } } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index 4f143fb87..796b28398 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -15,8 +15,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; +import java.util.ArrayList; +import java.util.List; import java.util.Random; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -24,6 +24,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatus; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatusResponse; +import com.databasepreservation.modules.siard.services.conversion.model.JobSubmissionResponse; +import com.databasepreservation.utils.ConfigUtils; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -38,9 +43,16 @@ public class HttpLobConversionService implements LobConversionService { private static final Logger log = LoggerFactory.getLogger(HttpLobConversionService.class); - private static final int MAX_NETWORK_RETRIES = 3; - private static final int BASE_POLLING_INTERVAL_MS = 2000; - private static final int MAX_POLLING_ATTEMPTS = 600; // ~20 minutes maximum wait per file before assuming Zombie Job + private static final int MAX_NETWORK_RETRIES = ConfigUtils.getProperty(3, + "dbptk.service.lob.conversion.networkRetries"); + private static final int BASE_POLLING_INTERVAL_MS = ConfigUtils.getProperty(2000, + "dbptk.service.lob.conversion.basePollingIntervalMs"); + private static final int MAX_POLLING_ATTEMPTS = ConfigUtils.getProperty(600, + "dbptk.service.lob.conversion.maxPollingAttempts"); + private static final int CONNECTION_TIMEOUT_SECONDS = ConfigUtils.getProperty(60, + "dbptk.service.lob.conversion.connectionTimeoutSeconds"); + + private static final String CRLF = "\r\n"; private final HttpClient httpClient; private final String baseUrl; @@ -54,7 +66,7 @@ public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTra this.targetFormat = targetFormat; this.fileTracker = fileTracker; this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(60)).build(); + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(CONNECTION_TIMEOUT_SECONDS)).build(); } @Override @@ -68,24 +80,20 @@ public ConversionResult convertLob(String cellId, InputStream inputStream) throw private String submitJob(String cellId, InputStream inputStream) throws Exception { String boundary = "DbptkBoundary" + System.currentTimeMillis(); - String header = "--" + boundary + "\r\n" + "Content-Disposition: form-data; name=\"targetFormat\"\r\n\r\n" - + targetFormat + "\r\n" + "--" + boundary + "\r\n" - + "Content-Disposition: form-data; name=\"file\"; filename=\"lob_" + cellId + ".bin\"\r\n" - + "Content-Type: application/octet-stream\r\n\r\n"; - - String footer = "\r\n--" + boundary + "--\r\n"; + String header = buildMultipartHeader(boundary, cellId); + String footer = CRLF + "--" + boundary + "--" + CRLF; InputStream headerStream = new ByteArrayInputStream(header.getBytes(StandardCharsets.UTF_8)); InputStream footerStream = new ByteArrayInputStream(footer.getBytes(StandardCharsets.UTF_8)); - InputStream multipartStream = new SequenceInputStream( - Collections.enumeration(Arrays.asList(headerStream, inputStream, footerStream))); + InputStream multipartStream = new SequenceInputStream(new SequenceInputStream(headerStream, inputStream), + footerStream); HttpRequest submitRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs")) .header("Content-Type", "multipart/form-data; boundary=" + boundary) .POST(BodyPublishers.ofInputStream(() -> multipartStream)).build(); - HttpResponse submitResponse = httpClient.send(submitRequest, BodyHandlers.ofString()); + HttpResponse submitResponse = executeWithRetry(submitRequest, BodyHandlers.ofString(), MAX_NETWORK_RETRIES); if (submitResponse.statusCode() >= 400) { log.error("API rejected LOB submission for cell {}. Status: {}, Body: {}", cellId, submitResponse.statusCode(), @@ -106,21 +114,23 @@ private void waitForCompletion(String cellId, String jobId) throws Exception { HttpResponse statusResponse = executeWithRetry(statusRequest, BodyHandlers.ofString(), MAX_NETWORK_RETRIES); - JobStatusResponse status = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); + JobStatusResponse response = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); - switch (status.status().toUpperCase()) { - case "COMPLETED", "DONE", "SUCCESS" -> { + switch (response.status()) { + case JobStatus.COMPLETED -> { log.debug("Job {} (Cell {}) completed successfully after {} attempts.", jobId, cellId, attempts); return; } - case "FAILED", "ERROR", "EVICTED" -> { - log.error("API reported terminal failure for Job {} (Cell {})", jobId, cellId); - throw new RuntimeException("Server failed to convert cell: " + cellId); + case JobStatus.FAILED, JobStatus.EVICTED -> { + log.error("API reported terminal failure for Job {} (Cell {}) with status: {}", jobId, cellId, + response.status()); + throw new RuntimeException( + "Server failed to convert cell: " + cellId + " (Status: " + response.status() + ")"); } - default -> { + case JobStatus.ACCEPTED, JobStatus.PROCESSING -> { if (attempts % 30 == 0) { log.warn("Job {} (Cell {}) is taking unusually long. Current status: {}. Attempt: {}/{}", jobId, cellId, - status.status(), attempts, MAX_POLLING_ATTEMPTS); + response.status(), attempts, MAX_POLLING_ATTEMPTS); } long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); Thread.sleep(sleepTime); @@ -157,35 +167,37 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); fileTracker.trackDir(extractionDir); - Path convertedLob = null; + Path normalizedExtractionDir = extractionDir.normalize(); + + List convertedFiles = new ArrayList<>(); Path reportFile = null; try (ZipInputStream zis = new ZipInputStream(new FileInputStream(tempZipFile.toFile()))) { ZipEntry zipEntry; while ((zipEntry = zis.getNextEntry()) != null) { - Path extractedFilePath = extractionDir.resolve(zipEntry.getName()); + Path extractedFilePath = extractionDir.resolve(zipEntry.getName()).normalize(); - if (!extractedFilePath.normalize().startsWith(extractionDir)) { - throw new SecurityException("Corrupted ZIP entry: " + zipEntry.getName()); + if (!extractedFilePath.startsWith(normalizedExtractionDir)) { + throw new SecurityException("Corrupted ZIP entry (Zip Slip vulnerability detected): " + zipEntry.getName()); } if (!zipEntry.isDirectory()) { - Files.copy(zis, extractedFilePath); + Files.copy(zis, extractedFilePath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); if (zipEntry.getName().toLowerCase().contains("report")) { reportFile = extractedFilePath; } else { - convertedLob = extractedFilePath; + convertedFiles.add(extractedFilePath); } } } } - if (convertedLob == null || reportFile == null) { - throw new RuntimeException("Downloaded ZIP lacks expected format (LOB + Report) for cell: " + cellId); + if (convertedFiles.isEmpty() || reportFile == null) { + throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); } - return new ConversionResult(convertedLob, reportFile); + return new ConversionResult(convertedFiles, reportFile); } private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, @@ -194,16 +206,47 @@ private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.B for (int attempt = 1; attempt <= maxRetries; attempt++) { try { - return httpClient.send(request, responseBodyHandler); + HttpResponse response = httpClient.send(request, responseBodyHandler); + + if (response.statusCode() >= 500) { + throw new IOException("Temporary server error: " + response.statusCode() + " - " + response.body()); + } + + return response; } catch (IOException e) { lastException = e; log.warn("Attempt {} failed for {}: {}", attempt, request.uri(), e.getMessage()); if (attempt == maxRetries) break; - Thread.sleep((long) Math.pow(2, attempt) * 1000); + + try { + Thread.sleep((long) Math.pow(2, attempt) * 1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Retry interrupted for URI: " + request.uri(), ie); + } } } throw new IOException("Exhausted all network retries for URI: " + request.uri(), lastException); } + + private String buildMultipartHeader(String boundary, String cellId) { + StringBuilder sb = new StringBuilder(); + + // Target format part header + sb.append("--").append(boundary).append(CRLF); + sb.append("Content-Disposition: form-data; name=\"targetFormat\"").append(CRLF); + sb.append(CRLF); + sb.append(this.targetFormat).append(CRLF); + + // File part header + sb.append("--").append(boundary).append(CRLF); + sb.append("Content-Disposition: form-data; name=\"file\"; filename=\"lob_").append(cellId).append(".bin\"") + .append(CRLF); + sb.append("Content-Type: application/octet-stream").append(CRLF); + sb.append(CRLF); + + return sb.toString(); + } } \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java index 038e779b4..8c7d8622a 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java @@ -2,7 +2,8 @@ import java.io.InputStream; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; + public interface LobConversionService { ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception; } - diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java similarity index 57% rename from dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java rename to dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java index 4b48a9bc5..1071de86f 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java @@ -1,9 +1,10 @@ -package com.databasepreservation.modules.siard.services.conversion; +package com.databasepreservation.modules.siard.services.conversion.model; import java.nio.file.Path; +import java.util.List; /** * @author Gabriel Barros */ -public record ConversionResult(Path convertedFile, Path reportFile) { +public record ConversionResult(List convertedFiles, Path reportFile) { } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java new file mode 100644 index 000000000..5e7107946 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +/** + * @author Gabriel Barros + */ +public enum JobStatus { + ACCEPTED, PROCESSING, COMPLETED, FAILED, EVICTED; +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java similarity index 65% rename from dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java rename to dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java index 79f8551c3..4028cd94f 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java @@ -1,7 +1,7 @@ -package com.databasepreservation.modules.siard.services.conversion; +package com.databasepreservation.modules.siard.services.conversion.model; /** * @author Gabriel Barros */ -public record JobStatusResponse(String status) { +public record JobStatusResponse(JobStatus status) { } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java similarity index 94% rename from dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java rename to dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java index 2abfaf6dd..933301871 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java @@ -1,4 +1,4 @@ -package com.databasepreservation.modules.siard.services.conversion; +package com.databasepreservation.modules.siard.services.conversion.model; /** * @author Gabriel Barros From f3a0c59ffc4a1a3310209e7ab2d88504d3438eec Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Mon, 15 Jun 2026 09:15:41 +0100 Subject: [PATCH 10/27] Add conversion service options to parameter categories and update input types --- .../databasepreservation/model/parameters/Parameter.java | 6 ++++-- .../modules/siard/SIARDDKModuleFactory.java | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java b/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java index 03f74e006..378250821 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java @@ -28,7 +28,7 @@ public enum INPUT_TYPE { /* GUI Helper for SIARD Export Module */ public enum CATEGORY_TYPE { - SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, NONE + SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, CONVERSION_SERVICE_OPTIONS, NONE } public enum FILE_FILTER_TYPE { @@ -312,7 +312,9 @@ public Parameter defaultSelectedIndex(Integer index) { return this; } - public Integer getDefaultSelectedIndex() { return defaultSelectedIndex; } + public Integer getDefaultSelectedIndex() { + return defaultSelectedIndex; + } /** * Gets the export option type for this parameter; Helper to automatize the diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java index f49f72f93..87e038dcd 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java @@ -176,11 +176,11 @@ public Parameters getExportModuleParameters() throws UnsupportedModuleException .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), - lobConversionEnabled.inputType(Parameter.INPUT_TYPE.COMBOBOX).possibleValues("true", "false") - .defaultSelectedIndex(1).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), - lobConversionEndpoint.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionEnabled.inputType(Parameter.INPUT_TYPE.CHECKBOX).exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS), + lobConversionEndpoint.inputType(Parameter.INPUT_TYPE.TEXT) + .exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS), lobConversionTargetFormat.inputType(Parameter.INPUT_TYPE.TEXT) - .exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS)), + .exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS)), null); } From 84d663181df9d4d7a9104df00ab50c0295b11671 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Tue, 16 Jun 2026 17:11:33 +0100 Subject: [PATCH 11/27] Adapt integration with lob conversion plugin to properly support binaries in siarddk --- .../content/SIARDDKContentExportStrategy.java | 123 ++++++++++-------- .../output/SIARDDKDatabaseExportModule.java | 4 +- .../SIARDDKContentPathExportStrategy.java | 11 +- .../conversion/HttpLobConversionService.java | 58 ++++++++- .../conversion/model/ConversionResult.java | 2 +- 5 files changed, 135 insertions(+), 63 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 9bb53fe9d..02fb84020 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -7,7 +7,6 @@ */ package com.databasepreservation.modules.siard.out.content; -import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStream; @@ -15,6 +14,9 @@ import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.IOUtils; @@ -36,7 +38,6 @@ import com.databasepreservation.model.structure.ColumnStructure; import com.databasepreservation.model.structure.SchemaStructure; import com.databasepreservation.model.structure.TableStructure; -import com.databasepreservation.modules.siard.common.LargeObject; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDConstants; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; @@ -45,6 +46,7 @@ import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import com.fasterxml.jackson.databind.ObjectMapper; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -320,60 +322,77 @@ public Row tableRow(Row row) throws ModuleException { final BinaryCell binaryCell = (BinaryCell) cell; - // BLOB is not NULL - - double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); - lobsTracker.addLOB(lobSizeMB); // Only if LOB not NULL - - // Determine the mimetype (Tika should use an inputstream which - // supports marks) - - InputStream is = new BufferedInputStream(binaryCell.createInputStream()); - // Removed because TIKA was a security vulnerability and this feature was not - // needed/not fully implemented (see #341) + double lobSizeTotal = 0; String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; - IOUtils.closeQuietly(is); - - // Archive BLOB - simultaneous writing always supported for - // SIARDDK - - tableXmlWriter.append(TAB).append(TAB).append("") - .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); - - String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); - String fileExtension; - if (mimetypeHandler.isMimetypeAllowed(mimeType)) { - fileExtension = mimetypeHandler.getFileExtension(mimeType); - } else { - fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; - // Log (table level) that unknown BLOB mimetype was detected - foundUnknownMimetype = true; - } - path += fileExtension; - LargeObject blob = new LargeObject(binaryCell, path); - - // Create new FileIndexFileStrategy - - // Write the BLOB - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), - writeStrategy); - InputStream in = blob.getInputStreamProvider().createInputStream(); - IOUtils.copy(in, out); - IOUtils.closeQuietly(in); - IOUtils.closeQuietly(out); - blob.getInputStreamProvider().cleanResources(); - - // Add file to docIndex (a lot easier to do here even though we - // are dealing with metadata) + if (mimeType.equals("application/zip")) { + // First pass to read report json + Map report = null; + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) { + ObjectMapper mapper = new ObjectMapper(); + report = mapper.readValue(zis.readAllBytes(), Map.class); + } + } + } + if (report == null) { + throw new ModuleException().withMessage( + "Could not find report in zip file for BLOB in table " + tableCounter + ", column " + columnIndex); + } + + List> processedArtifacts = ((List>) report + .get("processedArtifacts")); + + String originalFileName = processedArtifacts.getFirst().get("originalName"); + String firstProcessedFileMimeType = processedArtifacts.getFirst().get("finalFormat"); + + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + int fileCount = 0; + while ((zipEntry = zis.getNextEntry()) != null) { + // Archive BLOB - simultaneous writing always supported for + // SIARDDK + if (!zipEntry.getName().toLowerCase().contains("report")) { + fileCount++; + lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); + + String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); + Map processedArtifactReport = null; + for (Map artifact : processedArtifacts) { + if (artifact.get("finalFileName").equals(zipEntry.getName())) { + processedArtifactReport = artifact; + break; + } + } + String fileExtension = mimetypeHandler.getFileExtension(processedArtifactReport.get("finalFormat")); + outputPath += fileCount + "." + fileExtension; + + // Write the BLOB + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, + writeStrategy); + zis.transferTo(out); + + // Add file to fileIndex + SIARDDKFileIndexFileStrategy.addFile(outputPath); + } + } + } catch (IOException e) { + throw new ModuleException(); + } + lobsTracker.addLOB(lobSizeTotal); // Only if LOB not NULL + tableXmlWriter.append(TAB).append(TAB).append("") + .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); - // TO-DO: obtain (how?) hardcoded values - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - "originalFilename", fileExtension, null); + // Add file to docIndex (a lot easier to do here even though we + // are dealing with metadata) - // Add file to fileIndex - SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + // TO-DO: obtain (how?) hardcoded values + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + originalFileName, mimetypeHandler.getFileExtension(firstProcessedFileMimeType), null); + } } else { // never happens diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 08461e8a7..42d208a9e 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -359,8 +359,8 @@ private ProcessedRowContext processRowAsync(Row row) throws ModuleException { // TODO: Handle multiple files per cell if needed. Currently assumes single file // output. - Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFiles().getFirst()), - this.targetLobFormat); + Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.zipFile()), + "application/zip"); cells.set(i, newCell); // Track extracted parts to clean them individually later diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java index dcbb3fce0..98731d8f1 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java @@ -7,10 +7,11 @@ */ package com.databasepreservation.modules.siard.out.path; +import org.apache.commons.lang3.NotImplementedException; + import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.content.LOBsTracker; import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; -import org.apache.commons.lang3.NotImplementedException; /** * @author Andreas Kring @@ -24,14 +25,13 @@ public class SIARDDKContentPathExportStrategy implements ContentPathExportStrate private static final String SCHEMA_DIR = "schema"; private static final String DOCUMENT_DIR = "Documents"; private static final String DOC_COLLECTION = "docCollection"; - private static final String fileCount = "1"; // Design decision private LOBsTracker lobsTracker; public SIARDDKContentPathExportStrategy(LOBsTracker loBsTracker) { this.lobsTracker = loBsTracker; } - + public SIARDDKContentPathExportStrategy(SIARDDKExportModule siarddkExportModule) { this.lobsTracker = siarddkExportModule.getLobsTracker(); } @@ -71,13 +71,12 @@ public String getBlobFilePath(int schemaIndex, int tableIndex, int columnIndex, // TO-DO: add test case int docCollectionCount = lobsTracker.getDocCollectionCount(); - int LOBsCount = lobsTracker.getLOBsCount(); + int LOBsCount = lobsTracker.getLOBsCount() + 1; // Note: code assumes one file in each folder return new StringBuilder().append(DOCUMENT_DIR).append(SIARDDKConstants.FILE_SEPARATOR).append(DOC_COLLECTION) .append(docCollectionCount).append(SIARDDKConstants.FILE_SEPARATOR).append(LOBsCount) - .append(SIARDDKConstants.FILE_SEPARATOR).append(fileCount).append(SIARDDKConstants.FILE_EXTENSION_SEPARATOR) - .toString(); + .append(SIARDDKConstants.FILE_SEPARATOR).toString(); } // Not used in SIARDDK diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index 796b28398..5c8ffc780 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -74,7 +74,7 @@ public ConversionResult convertLob(String cellId, InputStream inputStream) throw log.debug("Initiating conversion pipeline for cell: {}", cellId); String jobId = submitJob(cellId, inputStream); waitForCompletion(cellId, jobId); - return downloadAndExtractResult(cellId, jobId); + return downloadResult(cellId, jobId); } private String submitJob(String cellId, InputStream inputStream) throws Exception { @@ -163,6 +163,23 @@ private ConversionResult downloadAndExtractResult(String cellId, String jobId) t } } + /** + * Downloads the resulting ZIP and lists its contents, returning the compressed + * file. + */ + private ConversionResult downloadResult(String cellId, String jobId) throws Exception { + Path zipFile = Files.createFile(Path.of("siarddk_conv_" + cellId + "_" + jobId + ".zip")); + // TODO: Should this still be tracked? + fileTracker.track(zipFile); + + HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) + .GET().build(); + + executeWithRetry(downloadRequest, BodyHandlers.ofFile(zipFile), MAX_NETWORK_RETRIES); + + return listZipContents(cellId, zipFile); + } + private ConversionResult extractZipContents(String cellId, Path tempZipFile) throws Exception { Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); fileTracker.trackDir(extractionDir); @@ -197,7 +214,44 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); } - return new ConversionResult(convertedFiles, reportFile); + return new ConversionResult(convertedFiles, reportFile, tempZipFile); + } + + private ConversionResult listZipContents(String cellId, Path zipFile) throws Exception { + Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); + fileTracker.trackDir(extractionDir); + + Path normalizedExtractionDir = extractionDir.normalize(); + + List convertedFiles = new ArrayList<>(); + Path reportFile = null; + + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile.toFile()))) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + Path extractedFilePath = extractionDir.resolve(zipEntry.getName()).normalize(); + + if (!extractedFilePath.startsWith(normalizedExtractionDir)) { + throw new SecurityException("Corrupted ZIP entry (Zip Slip vulnerability detected): " + zipEntry.getName()); + } + + if (!zipEntry.isDirectory()) { + Files.copy(zis, extractedFilePath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + + if (zipEntry.getName().toLowerCase().contains("report")) { + reportFile = extractedFilePath; + } else { + convertedFiles.add(extractedFilePath); + } + } + } + } + + if (convertedFiles.isEmpty() || reportFile == null) { + throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); + } + + return new ConversionResult(convertedFiles, reportFile, zipFile); } private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java index 1071de86f..5d229bb2d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java @@ -6,5 +6,5 @@ /** * @author Gabriel Barros */ -public record ConversionResult(List convertedFiles, Path reportFile) { +public record ConversionResult(List convertedFiles, Path reportFile, Path zipFile) { } From b55622f70a564f8407d5dec80814f9db7659a174 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Wed, 17 Jun 2026 15:43:00 +0100 Subject: [PATCH 12/27] Ignore invalid mimetypes and unprocessed files in SIARDDK export --- .../content/SIARDDKContentExportStrategy.java | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 02fb84020..89d182d42 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -13,6 +13,7 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; +import java.security.InvalidParameterException; import java.util.List; import java.util.Map; import java.util.zip.ZipEntry; @@ -354,11 +355,11 @@ public Row tableRow(Row row) throws ModuleException { while ((zipEntry = zis.getNextEntry()) != null) { // Archive BLOB - simultaneous writing always supported for // SIARDDK + + // Skip report file if (!zipEntry.getName().toLowerCase().contains("report")) { - fileCount++; - lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); - String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); + // Find processing report for current entry Map processedArtifactReport = null; for (Map artifact : processedArtifacts) { if (artifact.get("finalFileName").equals(zipEntry.getName())) { @@ -366,10 +367,31 @@ public Row tableRow(Row row) throws ModuleException { break; } } - String fileExtension = mimetypeHandler.getFileExtension(processedArtifactReport.get("finalFormat")); - outputPath += fileCount + "." + fileExtension; + if (processedArtifactReport == null) { + logger.warn( + "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", + zipEntry.getName(), tableCounter, columnIndex); + break; + } + + // Get processed file extension + String fileExtension = null; + try { + fileExtension = mimetypeHandler.getFileExtension(processedArtifactReport.get("finalFormat")); + } catch (InvalidParameterException e) { + logger.warn( + "Ignoring file {} in zip file for BLOB in table {}, column {} since it has an invalid mimetype.", + zipEntry.getName(), tableCounter, columnIndex); + break; + } + + // Increment file trackings + fileCount++; + lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); // Write the BLOB + String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); + outputPath += fileCount + "." + fileExtension; OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, writeStrategy); zis.transferTo(out); From 0a0549ef7aca131eaf891c279a2348c4d8970d45 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Wed, 17 Jun 2026 16:45:52 +0100 Subject: [PATCH 13/27] Default SIARDDK mimetype and processing status check --- .../out/content/SIARDDKContentExportStrategy.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 89d182d42..620bf3131 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -347,7 +347,8 @@ public Row tableRow(Row row) throws ModuleException { .get("processedArtifacts")); String originalFileName = processedArtifacts.getFirst().get("originalName"); - String firstProcessedFileMimeType = processedArtifacts.getFirst().get("finalFormat"); + // Default to tiff, attempt to find real mimetype as we go through zip entries + String processedFilesExtension = "tif"; try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { ZipEntry zipEntry; @@ -367,7 +368,7 @@ public Row tableRow(Row row) throws ModuleException { break; } } - if (processedArtifactReport == null) { + if (processedArtifactReport == null || !processedArtifactReport.get("status").equals("CONVERTED")) { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", zipEntry.getName(), tableCounter, columnIndex); @@ -385,6 +386,9 @@ public Row tableRow(Row row) throws ModuleException { break; } + // Set overall document mimetype + processedFilesExtension = fileExtension; + // Increment file trackings fileCount++; lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); @@ -413,7 +417,7 @@ public Row tableRow(Row row) throws ModuleException { // TO-DO: obtain (how?) hardcoded values SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - originalFileName, mimetypeHandler.getFileExtension(firstProcessedFileMimeType), null); + originalFileName, mimetypeHandler.getFileExtension(processedFilesExtension), null); } } else { From 1805f1b1acc7a9325d3a64a0a026123882625415 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Thu, 18 Jun 2026 10:16:32 +0100 Subject: [PATCH 14/27] Don't convert extension twice in SIARDDKContentExportStrategy --- .../content/SIARDDKContentExportStrategy.java | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 620bf3131..78ffc35b7 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -7,27 +7,6 @@ */ package com.databasepreservation.modules.siard.out.content; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; -import java.security.InvalidParameterException; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.IOUtils; -import org.jdom2.Document; -import org.jdom2.Element; -import org.jdom2.Namespace; -import org.jdom2.output.XMLOutputter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.ComposedCell; @@ -48,6 +27,26 @@ import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.IOUtils; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.Namespace; +import org.jdom2.output.XMLOutputter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.security.InvalidParameterException; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -417,7 +416,7 @@ public Row tableRow(Row row) throws ModuleException { // TO-DO: obtain (how?) hardcoded values SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - originalFileName, mimetypeHandler.getFileExtension(processedFilesExtension), null); + originalFileName, processedFilesExtension, null); } } else { From b7fab5915cc9b092bd28c8fa9e8705e6b9b9dc18 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Thu, 18 Jun 2026 10:23:36 +0100 Subject: [PATCH 15/27] Only write column data for lob columns if lobs are successfully processed --- .../content/SIARDDKContentExportStrategy.java | 70 ++++++++++--------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 78ffc35b7..e623e3a46 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -7,6 +7,27 @@ */ package com.databasepreservation.modules.siard.out.content; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.security.InvalidParameterException; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.IOUtils; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.Namespace; +import org.jdom2.output.XMLOutputter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.ComposedCell; @@ -27,26 +48,6 @@ import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.IOUtils; -import org.jdom2.Document; -import org.jdom2.Element; -import org.jdom2.Namespace; -import org.jdom2.output.XMLOutputter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; -import java.security.InvalidParameterException; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -349,9 +350,10 @@ public Row tableRow(Row row) throws ModuleException { // Default to tiff, attempt to find real mimetype as we go through zip entries String processedFilesExtension = "tif"; + int fileCount = 0; try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { ZipEntry zipEntry; - int fileCount = 0; + while ((zipEntry = zis.getNextEntry()) != null) { // Archive BLOB - simultaneous writing always supported for // SIARDDK @@ -371,7 +373,7 @@ public Row tableRow(Row row) throws ModuleException { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", zipEntry.getName(), tableCounter, columnIndex); - break; + continue; } // Get processed file extension @@ -382,7 +384,7 @@ public Row tableRow(Row row) throws ModuleException { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has an invalid mimetype.", zipEntry.getName(), tableCounter, columnIndex); - break; + continue; } // Set overall document mimetype @@ -407,16 +409,20 @@ public Row tableRow(Row row) throws ModuleException { throw new ModuleException(); } lobsTracker.addLOB(lobSizeTotal); // Only if LOB not NULL - tableXmlWriter.append(TAB).append(TAB).append("") - .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); - // Add file to docIndex (a lot easier to do here even though we - // are dealing with metadata) - - // TO-DO: obtain (how?) hardcoded values - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - originalFileName, processedFilesExtension, null); + if (fileCount > 0) { + tableXmlWriter.append(TAB).append(TAB).append("") + .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); + + // Add file to docIndex (a lot easier to do here even though we + // are dealing with metadata) + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, + lobsTracker.getDocCollectionCount(), originalFileName, processedFilesExtension, null); + } else { + tableXmlWriter.append(TAB).append(TAB).append("\n"); + } } } else { From c5e78fe034854706cd6dc28296895f0930b0d610 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Fri, 19 Jun 2026 14:28:21 +0100 Subject: [PATCH 16/27] Refactor HttpLobConversionService to improve exception handling and update method signatures for better error reporting --- .../content/SIARDDKContentExportStrategy.java | 19 +++++++---- .../output/SIARDDKDatabaseExportModule.java | 15 ++++++--- .../conversion/HttpLobConversionService.java | 22 ++++++++----- .../HttpLobConversionServiceException.java | 33 +++++++++++++++++++ .../conversion/LobConversionService.java | 5 ++- 5 files changed, 73 insertions(+), 21 deletions(-) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index e623e3a46..b810a754c 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -290,16 +290,14 @@ public Row tableRow(Row row) throws ModuleException { binaryCell.cleanResources(); } } else { - tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + whiteNilCell(columnIndex); } } else { // cell must contain BLOB or CLOB if (cell instanceof NullCell) { - tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + whiteNilCell(columnIndex); } else if (cell instanceof SimpleCell) { // CLOB is not NULL @@ -420,9 +418,13 @@ public Row tableRow(Row row) throws ModuleException { SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), originalFileName, processedFilesExtension, null); } else { - tableXmlWriter.append(TAB).append(TAB).append("\n"); + whiteNilCell(columnIndex); } + } else { + logger.warn( + "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", + mimeType, tableCounter, columnIndex); + whiteNilCell(columnIndex); } } else { @@ -440,6 +442,11 @@ public Row tableRow(Row row) throws ModuleException { return row; } + private void whiteNilCell(int columnIndex) throws IOException { + tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + } + @Override public void setOnceReporter(Reporter reporter) { this.reporter = reporter; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 42d208a9e..06b2e6b5b 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -43,6 +43,7 @@ import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionService; +import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionServiceException; import com.databasepreservation.modules.siard.services.conversion.LobConversionService; import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; @@ -367,13 +368,17 @@ private ProcessedRowContext processRowAsync(Row row) throws ModuleException { transientPaths.addAll(result.convertedFiles()); transientPaths.add(result.reportFile()); transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container - } catch (Exception e) { + } catch (IOException | InterruptedException | HttpLobConversionServiceException e) { + String statusCodeInfo = ""; + if (e instanceof HttpLobConversionServiceException apiEx && apiEx.getHttpStatusCode() != null) { + statusCodeInfo = " (HTTP " + apiEx.getHttpStatusCode() + ")"; + } + String errorMsg = String.format( - "Conversion failed for cell '%s' in row %d. " - + "Please check if the LOB service is running and accessible. Detail: %s", - cell.getId(), row.getIndex(), e.getMessage()); + "Conversion failed for cell '%s' in row %d%s. " + "Pipeline continuing with original file. Detail: %s", + cell.getId(), row.getIndex(), statusCodeInfo, e.getMessage()); + logger.error(errorMsg); - throw new ModuleException().withMessage(errorMsg).withCause(e); } } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index 5c8ffc780..83256da38 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -70,14 +70,16 @@ public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTra } @Override - public ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception { + public ConversionResult convertLob(String cellId, InputStream inputStream) + throws IOException, InterruptedException, HttpLobConversionServiceException { log.debug("Initiating conversion pipeline for cell: {}", cellId); String jobId = submitJob(cellId, inputStream); waitForCompletion(cellId, jobId); return downloadResult(cellId, jobId); } - private String submitJob(String cellId, InputStream inputStream) throws Exception { + private String submitJob(String cellId, InputStream inputStream) + throws IOException, InterruptedException, HttpLobConversionServiceException { String boundary = "DbptkBoundary" + System.currentTimeMillis(); String header = buildMultipartHeader(boundary, cellId); @@ -98,7 +100,8 @@ private String submitJob(String cellId, InputStream inputStream) throws Exceptio if (submitResponse.statusCode() >= 400) { log.error("API rejected LOB submission for cell {}. Status: {}, Body: {}", cellId, submitResponse.statusCode(), submitResponse.body()); - throw new RuntimeException("Failed to submit LOB for cell " + cellId); + throw new HttpLobConversionServiceException("Failed to submit LOB for cell " + cellId, + submitResponse.statusCode()); } JobSubmissionResponse job = objectMapper.readValue(submitResponse.body(), JobSubmissionResponse.class); @@ -106,7 +109,8 @@ private String submitJob(String cellId, InputStream inputStream) throws Exceptio return job.id(); } - private void waitForCompletion(String cellId, String jobId) throws Exception { + private void waitForCompletion(String cellId, String jobId) + throws IOException, InterruptedException, HttpLobConversionServiceException { log.debug("Awaiting completion of Job {} (Cell {})", jobId, cellId); for (int attempts = 1; attempts <= MAX_POLLING_ATTEMPTS; attempts++) { @@ -124,7 +128,7 @@ private void waitForCompletion(String cellId, String jobId) throws Exception { case JobStatus.FAILED, JobStatus.EVICTED -> { log.error("API reported terminal failure for Job {} (Cell {}) with status: {}", jobId, cellId, response.status()); - throw new RuntimeException( + throw new HttpLobConversionServiceException( "Server failed to convert cell: " + cellId + " (Status: " + response.status() + ")"); } case JobStatus.ACCEPTED, JobStatus.PROCESSING -> { @@ -139,7 +143,7 @@ private void waitForCompletion(String cellId, String jobId) throws Exception { } log.error("Zombie Job detected. API failed to resolve Job {} (Cell {}) within the maximum polling threshold.", jobId, cellId); - throw new RuntimeException("Timeout after waiting for conversion of cell: " + cellId); + throw new HttpLobConversionServiceException("Timeout after waiting for conversion of cell: " + cellId); } /** @@ -167,7 +171,7 @@ private ConversionResult downloadAndExtractResult(String cellId, String jobId) t * Downloads the resulting ZIP and lists its contents, returning the compressed * file. */ - private ConversionResult downloadResult(String cellId, String jobId) throws Exception { + private ConversionResult downloadResult(String cellId, String jobId) throws IOException, InterruptedException { Path zipFile = Files.createFile(Path.of("siarddk_conv_" + cellId + "_" + jobId + ".zip")); // TODO: Should this still be tracked? fileTracker.track(zipFile); @@ -217,7 +221,7 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr return new ConversionResult(convertedFiles, reportFile, tempZipFile); } - private ConversionResult listZipContents(String cellId, Path zipFile) throws Exception { + private ConversionResult listZipContents(String cellId, Path zipFile) throws IOException { Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); fileTracker.trackDir(extractionDir); @@ -255,7 +259,7 @@ private ConversionResult listZipContents(String cellId, Path zipFile) throws Exc } private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, - int maxRetries) throws Exception { + int maxRetries) throws InterruptedException, IOException { Exception lastException = null; for (int attempt = 1; attempt <= maxRetries; attempt++) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java new file mode 100644 index 000000000..9fc7d9960 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java @@ -0,0 +1,33 @@ +package com.databasepreservation.modules.siard.services.conversion; + +/** + * @author Gabriel Barros + */ +public class HttpLobConversionServiceException extends Exception { + + private final Integer httpStatusCode; + + public HttpLobConversionServiceException(String message) { + super(message); + this.httpStatusCode = null; + } + + public HttpLobConversionServiceException(String message, Throwable cause) { + super(message, cause); + this.httpStatusCode = null; + } + + public HttpLobConversionServiceException(String message, int httpStatusCode) { + super(message); + this.httpStatusCode = httpStatusCode; + } + + public HttpLobConversionServiceException(String message, int httpStatusCode, Throwable cause) { + super(message, cause); + this.httpStatusCode = httpStatusCode; + } + + public Integer getHttpStatusCode() { + return httpStatusCode; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java index 8c7d8622a..0f26c5b32 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java @@ -1,9 +1,12 @@ package com.databasepreservation.modules.siard.services.conversion; +import java.io.IOException; import java.io.InputStream; +import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; public interface LobConversionService { - ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception; + ConversionResult convertLob(String cellId, InputStream inputStream) + throws IOException, ModuleException, InterruptedException, HttpLobConversionServiceException; } From 233fce242e0c85c983ac77a670c72218f9fd10f4 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Sun, 21 Jun 2026 11:10:41 +0100 Subject: [PATCH 17/27] Refactor SIARDDKContentExportStrategy to improve artifact handling and update file processing logic --- .../content/SIARDDKContentExportStrategy.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index b810a754c..9996d93da 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -341,10 +341,9 @@ public Row tableRow(Row row) throws ModuleException { "Could not find report in zip file for BLOB in table " + tableCounter + ", column " + columnIndex); } - List> processedArtifacts = ((List>) report - .get("processedArtifacts")); + List> artifacts = (List>) report.get("artifacts"); - String originalFileName = processedArtifacts.getFirst().get("originalName"); + String originalFileName = (String) report.get("originalInputFile"); // Default to tiff, attempt to find real mimetype as we go through zip entries String processedFilesExtension = "tif"; @@ -360,14 +359,16 @@ public Row tableRow(Row row) throws ModuleException { if (!zipEntry.getName().toLowerCase().contains("report")) { // Find processing report for current entry - Map processedArtifactReport = null; - for (Map artifact : processedArtifacts) { - if (artifact.get("finalFileName").equals(zipEntry.getName())) { + Map processedArtifactReport = null; + for (Map artifact : artifacts) { + if (artifact.get("logicalName").equals(zipEntry.getName())) { processedArtifactReport = artifact; break; } } - if (processedArtifactReport == null || !processedArtifactReport.get("status").equals("CONVERTED")) { + boolean isBypassed = processedArtifactReport != null ? (Boolean) processedArtifactReport.get("isBypassed") : true; + + if (processedArtifactReport == null || isBypassed) { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", zipEntry.getName(), tableCounter, columnIndex); @@ -377,7 +378,8 @@ public Row tableRow(Row row) throws ModuleException { // Get processed file extension String fileExtension = null; try { - fileExtension = mimetypeHandler.getFileExtension(processedArtifactReport.get("finalFormat")); + String finalMimeType = (String) processedArtifactReport.get("finalMimeType"); + fileExtension = mimetypeHandler.getFileExtension(finalMimeType); } catch (InvalidParameterException e) { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has an invalid mimetype.", From 34209662656b2c02b14721ab330e9a41ec65dcc9 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Sun, 21 Jun 2026 15:16:49 +0100 Subject: [PATCH 18/27] Enhance SIARDDKContentExportStrategy with LOB conversion auditing and report extraction --- .../content/SIARDDKContentExportStrategy.java | 208 +++++++++--------- .../conversion/LobConversionAuditor.java | 41 ++++ .../model/report/ArtifactReport.java | 13 ++ .../model/report/AuditTrailStep.java | 13 ++ .../model/report/ConversionReport.java | 17 ++ .../conversion/model/report/DbptkContext.java | 9 + 6 files changed, 198 insertions(+), 103 deletions(-) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 9996d93da..a796e4086 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -13,9 +13,9 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; -import java.security.InvalidParameterException; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -47,6 +47,10 @@ import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import com.databasepreservation.modules.siard.services.conversion.LobConversionAuditor; +import com.databasepreservation.modules.siard.services.conversion.model.report.ArtifactReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.DbptkContext; import com.fasterxml.jackson.databind.ObjectMapper; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -72,6 +76,9 @@ public class SIARDDKContentExportStrategy implements ContentExportStrategy { private final LOBsTracker lobsTracker; private final MimetypeHandler mimetypeHandler; + private final LobConversionAuditor auditor; + private final ObjectMapper mapper; + private Reporter reporter; public SIARDDKContentExportStrategy(SIARDDKExportModule siarddkExportModule) { @@ -88,6 +95,11 @@ public SIARDDKContentExportStrategy(SIARDDKExportModule siarddkExportModule) { baseContainer = siarddkExportModule.getMainContainer(); writeStrategy = siarddkExportModule.getWriteStrategy(); lobsTracker = siarddkExportModule.getLobsTracker(); + + this.mapper = new ObjectMapper(); + Path exportRoot = baseContainer.getPath().getParent(); + String archiveName = baseContainer.getPath().getFileName().toString(); + this.auditor = new LobConversionAuditor(exportRoot, archiveName); } @Override @@ -318,117 +330,20 @@ public Row tableRow(Row row) throws ModuleException { } else if (cell instanceof BinaryCell) { // BLOB case - final BinaryCell binaryCell = (BinaryCell) cell; - - double lobSizeTotal = 0; String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; + // ------------------------------------------------------------- + // BLOB EXTRACTION DELEGATION + // ------------------------------------------------------------- if (mimeType.equals("application/zip")) { - // First pass to read report json - Map report = null; - try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { - ZipEntry zipEntry; - while ((zipEntry = zis.getNextEntry()) != null) { - if (zipEntry.getName().toLowerCase().contains("report")) { - ObjectMapper mapper = new ObjectMapper(); - report = mapper.readValue(zis.readAllBytes(), Map.class); - } - } - } - if (report == null) { - throw new ModuleException().withMessage( - "Could not find report in zip file for BLOB in table " + tableCounter + ", column " + columnIndex); - } - - List> artifacts = (List>) report.get("artifacts"); - - String originalFileName = (String) report.get("originalInputFile"); - // Default to tiff, attempt to find real mimetype as we go through zip entries - String processedFilesExtension = "tif"; - - int fileCount = 0; - try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { - ZipEntry zipEntry; - - while ((zipEntry = zis.getNextEntry()) != null) { - // Archive BLOB - simultaneous writing always supported for - // SIARDDK - - // Skip report file - if (!zipEntry.getName().toLowerCase().contains("report")) { - - // Find processing report for current entry - Map processedArtifactReport = null; - for (Map artifact : artifacts) { - if (artifact.get("logicalName").equals(zipEntry.getName())) { - processedArtifactReport = artifact; - break; - } - } - boolean isBypassed = processedArtifactReport != null ? (Boolean) processedArtifactReport.get("isBypassed") : true; - - if (processedArtifactReport == null || isBypassed) { - logger.warn( - "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", - zipEntry.getName(), tableCounter, columnIndex); - continue; - } - - // Get processed file extension - String fileExtension = null; - try { - String finalMimeType = (String) processedArtifactReport.get("finalMimeType"); - fileExtension = mimetypeHandler.getFileExtension(finalMimeType); - } catch (InvalidParameterException e) { - logger.warn( - "Ignoring file {} in zip file for BLOB in table {}, column {} since it has an invalid mimetype.", - zipEntry.getName(), tableCounter, columnIndex); - continue; - } - - // Set overall document mimetype - processedFilesExtension = fileExtension; - - // Increment file trackings - fileCount++; - lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); - - // Write the BLOB - String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); - outputPath += fileCount + "." + fileExtension; - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, - writeStrategy); - zis.transferTo(out); - - // Add file to fileIndex - SIARDDKFileIndexFileStrategy.addFile(outputPath); - } - } - } catch (IOException e) { - throw new ModuleException(); - } - lobsTracker.addLOB(lobSizeTotal); // Only if LOB not NULL - - if (fileCount > 0) { - tableXmlWriter.append(TAB).append(TAB).append("") - .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); - - // Add file to docIndex (a lot easier to do here even though we - // are dealing with metadata) - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, - lobsTracker.getDocCollectionCount(), originalFileName, processedFilesExtension, null); - } else { - whiteNilCell(columnIndex); - } + processConvertedLobArchive(binaryCell, columnIndex); } else { logger.warn( "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", mimeType, tableCounter, columnIndex); whiteNilCell(columnIndex); } - } else { // never happens } @@ -444,6 +359,93 @@ public Row tableRow(Row row) throws ModuleException { return row; } + private void processConvertedLobArchive(BinaryCell binaryCell, int columnIndex) throws ModuleException { + try { + ConversionReport report = extractReportFromZip(binaryCell); + if (report == null) { + throw new ModuleException().withMessage("Missing conversion_report.json in cell archive."); + } + + List siardPhysicalPaths = new ArrayList<>(); + int fileCount = 0; + String processedFilesExtension = "tif"; + + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) + continue; + + ArtifactReport artifactMeta = findArtifactMetadata(report.artifacts(), zipEntry.getName()); + + if (artifactMeta == null || artifactMeta.isBypassed()) { + logger.warn("Ignoring bypassed or unknown file: {}. Reason: {}", zipEntry.getName(), + artifactMeta != null ? artifactMeta.errorMessage() : "Not in report"); + continue; + } + + String fileExt = mimetypeHandler.getFileExtension(artifactMeta.finalMimeType()); + processedFilesExtension = fileExt; + fileCount++; + + String outputPath = writeLobToSiardStorage(zis, fileCount, fileExt); + siardPhysicalPaths.add(outputPath); + } + } + + double lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); + lobsTracker.addLOB(lobSizeTotal); + + if (fileCount > 0) { + writeLobReferenceToXml(columnIndex); + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + report.originalInputFile(), processedFilesExtension, null); + } else { + whiteNilCell(columnIndex); + } + + // Enriquecimento e Auditoria + ConversionReport enrichedReport = report + .withContext(new DbptkContext(tableCounter, columnIndex, siardPhysicalPaths)); + auditor.appendAuditRecord(enrichedReport); + + } catch (Exception e) { + throw new ModuleException().withMessage("Failed to process converted ZIP archive").withCause(e); + } + } + + private ConversionReport extractReportFromZip(BinaryCell binaryCell) throws Exception { + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) { + return mapper.readValue(zis.readAllBytes(), ConversionReport.class); + } + } + } + return null; + } + + private ArtifactReport findArtifactMetadata(List artifacts, String fileName) { + if (artifacts == null) + return null; + return artifacts.stream().filter(a -> fileName.equals(a.logicalName())).findFirst().orElse(null); + } + + private String writeLobToSiardStorage(InputStream zis, int fileCount, String extension) throws Exception { + String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + fileCount + "." + extension; + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, writeStrategy); + zis.transferTo(out); + SIARDDKFileIndexFileStrategy.addFile(outputPath); + return outputPath; + } + + private void writeLobReferenceToXml(int columnIndex) throws IOException { + tableXmlWriter.append(TAB).append(TAB).append("") + .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); + } + private void whiteNilCell(int columnIndex) throws IOException { tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java new file mode 100644 index 000000000..a5d4d30d1 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java @@ -0,0 +1,41 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Enriches and persists LOB conversion reports. + * + * @author Gabriel Barros + */ +public class LobConversionAuditor { + private static final Logger logger = LoggerFactory.getLogger(LobConversionAuditor.class); + private final ObjectMapper mapper; + private final Path auditFilePath; + + public LobConversionAuditor(Path baseExportDirectory, String archiveName) { + this.mapper = new ObjectMapper(); + String fileName = archiveName + "_lob_conversion_audit.jsonl"; + this.auditFilePath = baseExportDirectory.resolve(fileName); + } + + public void appendAuditRecord(ConversionReport report) { + if (report == null) + return; + + try { + String jsonLine = mapper.writeValueAsString(report); + Files.writeString(auditFilePath, jsonLine + System.lineSeparator(), StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (Exception e) { + logger.error("Failed to append conversion report to audit log.", e); + } + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java new file mode 100644 index 000000000..c774cfd33 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java @@ -0,0 +1,13 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record ArtifactReport(String logicalName, String originalMimeType, String finalMimeType, boolean isBypassed, + List formatHistory, List auditTrail, String errorMessage) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java new file mode 100644 index 000000000..0c1307ab7 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java @@ -0,0 +1,13 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record AuditTrailStep(String stepId, String pluginId, String agentName, String agentVersion, String agentType, + String command, Map parameters, long durationMs, boolean successful, String errorMessage) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java new file mode 100644 index 000000000..363d24e53 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java @@ -0,0 +1,17 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record ConversionReport(String jobId, String status, String originalInputFile, Integer totalArtifactsProduced, + List artifacts, String errorMessage, DbptkContext dbptkContext) { + public ConversionReport withContext(DbptkContext context) { + return new ConversionReport(jobId, status, originalInputFile, totalArtifactsProduced, artifacts, errorMessage, + context); + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java new file mode 100644 index 000000000..bf83dee64 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java @@ -0,0 +1,9 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +/** + * @author Gabriel Barros + */ +public record DbptkContext(int tableIndex, int columnIndex, List siardPaths) { +} From e568583f23fedabde43c7b4401dd86af6c76ffe0 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Mon, 22 Jun 2026 11:04:21 +0100 Subject: [PATCH 19/27] Add ComplianceStatus enum and enhance report models with additional properties --- .../content/SIARDDKContentExportStrategy.java | 10 +-- .../output/SIARDDKDatabaseExportModule.java | 5 +- .../conversion/HttpLobConversionService.java | 62 +------------------ .../model/report/ArtifactReport.java | 8 ++- .../model/report/AuditTrailStep.java | 8 ++- .../model/report/ComplianceStatus.java | 8 +++ .../model/report/ConversionReport.java | 11 +++- .../conversion/model/report/DbptkContext.java | 5 +- 8 files changed, 43 insertions(+), 74 deletions(-) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index a796e4086..d5f227166 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -337,7 +337,7 @@ public Row tableRow(Row row) throws ModuleException { // BLOB EXTRACTION DELEGATION // ------------------------------------------------------------- if (mimeType.equals("application/zip")) { - processConvertedLobArchive(binaryCell, columnIndex); + processConvertedLobArchive(binaryCell, row.getIndex(), columnIndex); } else { logger.warn( "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", @@ -359,7 +359,8 @@ public Row tableRow(Row row) throws ModuleException { return row; } - private void processConvertedLobArchive(BinaryCell binaryCell, int columnIndex) throws ModuleException { + private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, int columnIndex) + throws ModuleException { try { ConversionReport report = extractReportFromZip(binaryCell); if (report == null) { @@ -399,14 +400,13 @@ private void processConvertedLobArchive(BinaryCell binaryCell, int columnIndex) if (fileCount > 0) { writeLobReferenceToXml(columnIndex); SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - report.originalInputFile(), processedFilesExtension, null); + report.originalFilename(), processedFilesExtension, null); } else { whiteNilCell(columnIndex); } - // Enriquecimento e Auditoria ConversionReport enrichedReport = report - .withContext(new DbptkContext(tableCounter, columnIndex, siardPhysicalPaths)); + .withContext(new DbptkContext(tableCounter, rowIndex, columnIndex, siardPhysicalPaths)); auditor.appendAuditRecord(enrichedReport); } catch (Exception e) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 06b2e6b5b..9fd917c1e 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -367,7 +367,10 @@ private ProcessedRowContext processRowAsync(Row row) throws ModuleException { // Track extracted parts to clean them individually later transientPaths.addAll(result.convertedFiles()); transientPaths.add(result.reportFile()); - transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container + transientPaths.add(result.zipFile()); + if (!result.convertedFiles().isEmpty()) { + transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container + } } catch (IOException | InterruptedException | HttpLobConversionServiceException e) { String statusCodeInfo = ""; if (e instanceof HttpLobConversionServiceException apiEx && apiEx.getHttpStatusCode() != null) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index 83256da38..bf20cadd6 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -146,34 +146,13 @@ private void waitForCompletion(String cellId, String jobId) throw new HttpLobConversionServiceException("Timeout after waiting for conversion of cell: " + cellId); } - /** - * Downloads the resulting ZIP and extracts its contents, freeing the ZIP file - * immediately after. - */ - private ConversionResult downloadAndExtractResult(String cellId, String jobId) throws Exception { - Path tempZipFile = Files.createTempFile("siarddk_conv_" + cellId + "_", ".zip"); - fileTracker.track(tempZipFile); - - try { - HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) - .GET().build(); - - executeWithRetry(downloadRequest, BodyHandlers.ofFile(tempZipFile), MAX_NETWORK_RETRIES); - - return extractZipContents(cellId, tempZipFile); - } finally { - // Free disk space immediately after extraction - fileTracker.deleteEarly(tempZipFile); - } - } - /** * Downloads the resulting ZIP and lists its contents, returning the compressed * file. */ private ConversionResult downloadResult(String cellId, String jobId) throws IOException, InterruptedException { - Path zipFile = Files.createFile(Path.of("siarddk_conv_" + cellId + "_" + jobId + ".zip")); - // TODO: Should this still be tracked? + Path zipFile = Files.createTempFile("siarddk_conv_" + cellId + "_" + jobId, ".zip"); + fileTracker.track(zipFile); HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) @@ -184,43 +163,6 @@ private ConversionResult downloadResult(String cellId, String jobId) throws IOEx return listZipContents(cellId, zipFile); } - private ConversionResult extractZipContents(String cellId, Path tempZipFile) throws Exception { - Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); - fileTracker.trackDir(extractionDir); - - Path normalizedExtractionDir = extractionDir.normalize(); - - List convertedFiles = new ArrayList<>(); - Path reportFile = null; - - try (ZipInputStream zis = new ZipInputStream(new FileInputStream(tempZipFile.toFile()))) { - ZipEntry zipEntry; - while ((zipEntry = zis.getNextEntry()) != null) { - Path extractedFilePath = extractionDir.resolve(zipEntry.getName()).normalize(); - - if (!extractedFilePath.startsWith(normalizedExtractionDir)) { - throw new SecurityException("Corrupted ZIP entry (Zip Slip vulnerability detected): " + zipEntry.getName()); - } - - if (!zipEntry.isDirectory()) { - Files.copy(zis, extractedFilePath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); - - if (zipEntry.getName().toLowerCase().contains("report")) { - reportFile = extractedFilePath; - } else { - convertedFiles.add(extractedFilePath); - } - } - } - } - - if (convertedFiles.isEmpty() || reportFile == null) { - throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); - } - - return new ConversionResult(convertedFiles, reportFile, tempZipFile); - } - private ConversionResult listZipContents(String cellId, Path zipFile) throws IOException { Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); fileTracker.trackDir(extractionDir); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java index c774cfd33..80552d0b5 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java @@ -3,11 +3,15 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; /** * @author Gabriel Barros */ @JsonIgnoreProperties(ignoreUnknown = true) -public record ArtifactReport(String logicalName, String originalMimeType, String finalMimeType, boolean isBypassed, - List formatHistory, List auditTrail, String errorMessage) { +public record ArtifactReport(@JsonProperty("logicalName") String logicalName, + @JsonProperty("originalMimeType") String originalMimeType, @JsonProperty("finalMimeType") String finalMimeType, + @JsonProperty("isBypassed") boolean isBypassed, @JsonProperty("complianceStatus") ComplianceStatus complianceStatus, + @JsonProperty("formatHistory") List formatHistory, + @JsonProperty("auditTrail") List auditTrail, @JsonProperty("errorMessage") String errorMessage) { } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java index 0c1307ab7..eab47c711 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java @@ -3,11 +3,15 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; /** * @author Gabriel Barros */ @JsonIgnoreProperties(ignoreUnknown = true) -public record AuditTrailStep(String stepId, String pluginId, String agentName, String agentVersion, String agentType, - String command, Map parameters, long durationMs, boolean successful, String errorMessage) { +public record AuditTrailStep(@JsonProperty("stepId") String stepId, @JsonProperty("pluginId") String pluginId, + @JsonProperty("agentName") String agentName, @JsonProperty("agentVersion") String agentVersion, + @JsonProperty("agentType") String agentType, @JsonProperty("command") String command, + @JsonProperty("parameters") Map parameters, @JsonProperty("durationMs") long durationMs, + @JsonProperty("successful") boolean successful, @JsonProperty("errorMessage") String errorMessage) { } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java new file mode 100644 index 000000000..8d484c56a --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +/** + * @author Gabriel Barros + */ +public enum ComplianceStatus { + PASSED, PARTIAL, FAILED +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java index 363d24e53..4deafef2c 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java @@ -2,16 +2,21 @@ import java.util.List; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatus; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; /** * @author Gabriel Barros */ @JsonIgnoreProperties(ignoreUnknown = true) -public record ConversionReport(String jobId, String status, String originalInputFile, Integer totalArtifactsProduced, - List artifacts, String errorMessage, DbptkContext dbptkContext) { +public record ConversionReport(@JsonProperty("jobId") String jobId, @JsonProperty("status") JobStatus status, + @JsonProperty("originalFilename") String originalFilename, + @JsonProperty("totalArtifactsProduced") Integer totalArtifactsProduced, + @JsonProperty("artifacts") List artifacts, @JsonProperty("errorMessage") String errorMessage, + @JsonProperty("dbptkContext") DbptkContext dbptkContext) { public ConversionReport withContext(DbptkContext context) { - return new ConversionReport(jobId, status, originalInputFile, totalArtifactsProduced, artifacts, errorMessage, + return new ConversionReport(jobId, status, originalFilename, totalArtifactsProduced, artifacts, errorMessage, context); } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java index bf83dee64..900387b56 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java @@ -2,8 +2,11 @@ import java.util.List; +import com.fasterxml.jackson.annotation.JsonProperty; + /** * @author Gabriel Barros */ -public record DbptkContext(int tableIndex, int columnIndex, List siardPaths) { +public record DbptkContext(@JsonProperty("tableIndex") int tableIndex, @JsonProperty("rowIndex") long rowIndex, + @JsonProperty("columnIndex") int columnIndex, @JsonProperty("siardPaths") List siardPaths) { } From eb74b3834563fe93242e79ff590f5bb1d2d67a3f Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Mon, 22 Jun 2026 15:58:02 +0100 Subject: [PATCH 20/27] Include researchIndex.xsd in produced SIARDDK 128 --- .../siard/common/path/SIARDDKMetadataPathStrategy.java | 7 ++++--- .../modules/siard/constants/SIARDDKConstants.java | 1 + .../siard/out/metadata/SIARDDKMetadataExportStrategy.java | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java index a4fe44a1a..f0a445522 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java @@ -7,12 +7,12 @@ */ package com.databasepreservation.modules.siard.common.path; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; - import java.security.InvalidParameterException; import java.util.ArrayList; import java.util.Arrays; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; + /** * @author Andreas Kring * @@ -54,7 +54,8 @@ public boolean checkFilename(String filename) { // Valid filenames String[] validFileNames = {SIARDDKConstants.TABLE_INDEX, SIARDDKConstants.ARCHIVE_INDEX, SIARDDKConstants.DOC_INDEX, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, SIARDDKConstants.FILE_INDEX, - SIARDDKConstants.DOCUMENT_IDENTIFICATION, SIARDDKConstants.XML_SCHEMA, "fileIndex_original", "docIndex_original"}; + SIARDDKConstants.DOCUMENT_IDENTIFICATION, SIARDDKConstants.XML_SCHEMA, SIARDDKConstants.RESEARCH_INDEX, + "fileIndex_original", "docIndex_original"}; ArrayList validFilenames = new ArrayList(Arrays.asList(validFileNames)); if (validFilenames.contains(filename)) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java index 68dd9d90b..203433413 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java @@ -54,6 +54,7 @@ public class SIARDDKConstants { public static final String ARCHIVE_INDEX = "archiveIndex"; public static final String TABLE_INDEX = "tableIndex"; public static final String FILE_INDEX = "fileIndex"; + public static final String RESEARCH_INDEX = "researchIndex"; public static final String DOC_INDEX = "docIndex"; public static final String DOCUMENT_IDENTIFICATION = "documentIdentification"; public static final String XML_SCHEMA = "XMLSchema"; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java index 7fee0e106..f728ac0d8 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java @@ -154,6 +154,7 @@ public void writeMetadataXSD(DatabaseStructure dbStructure, SIARDArchiveContaine writeSchemaFile(outputContainer, SIARDDKConstants.ARCHIVE_INDEX, writeStrategy); writeSchemaFile(outputContainer, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, writeStrategy); writeSchemaFile(outputContainer, SIARDDKConstants.FILE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.RESEARCH_INDEX, writeStrategy); if (lobsTracker.getLOBsCount() > 0) { writeSchemaFile(outputContainer, SIARDDKConstants.DOC_INDEX, writeStrategy); } From a9f162e3565fa10e0c204905802f4176cde62c69 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Tue, 23 Jun 2026 09:18:39 +0100 Subject: [PATCH 21/27] Only include researchIndex.xsd in SIARDDK128 --- .../SIARDDK128MetadataExportStrategy.java | 16 ++++++++++++ .../SIARDDKMetadataExportStrategy.java | 26 +++++++++---------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java index 4a0b82c5f..5b28bd2d3 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java @@ -106,4 +106,20 @@ public void writeMetadataXML(DatabaseStructure dbStructure, SIARDArchiveContaine createLocalSharedFolder(outputContainer); } + + @Override + public void writeMetadataXSD(DatabaseStructure dbStructure, SIARDArchiveContainer outputContainer, + WriteStrategy writeStrategy) throws ModuleException { + + // Write contents to Schemas/standard + writeSchemaFile(outputContainer, SIARDDKConstants.XML_SCHEMA, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.TABLE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.ARCHIVE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.FILE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.RESEARCH_INDEX, writeStrategy); + if (lobsTracker.getLOBsCount() > 0) { + writeSchemaFile(outputContainer, SIARDDKConstants.DOC_INDEX, writeStrategy); + } + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java index f728ac0d8..53de790e0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java @@ -7,6 +7,18 @@ */ package com.databasepreservation.modules.siard.out.metadata; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.reporters.Reporter; import com.databasepreservation.model.structure.DatabaseStructure; @@ -17,17 +29,6 @@ import com.databasepreservation.modules.siard.out.content.LOBsTracker; import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.write.WriteStrategy; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; /** * @author Andreas Kring @@ -154,7 +155,6 @@ public void writeMetadataXSD(DatabaseStructure dbStructure, SIARDArchiveContaine writeSchemaFile(outputContainer, SIARDDKConstants.ARCHIVE_INDEX, writeStrategy); writeSchemaFile(outputContainer, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, writeStrategy); writeSchemaFile(outputContainer, SIARDDKConstants.FILE_INDEX, writeStrategy); - writeSchemaFile(outputContainer, SIARDDKConstants.RESEARCH_INDEX, writeStrategy); if (lobsTracker.getLOBsCount() > 0) { writeSchemaFile(outputContainer, SIARDDKConstants.DOC_INDEX, writeStrategy); } @@ -165,7 +165,7 @@ public void setOnceReporter(Reporter reporter) { this.reporter = reporter; } - private void writeSchemaFile(SIARDArchiveContainer container, String indexFile, WriteStrategy writeStrategy) + protected void writeSchemaFile(SIARDArchiveContainer container, String indexFile, WriteStrategy writeStrategy) throws ModuleException { InputStream inputStream = this.getClass().getResourceAsStream(metadataPathStrategy.getXsdResourcePath(indexFile)); From 3925b4c99a3d8850003254584526e41481fe27ab Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Mon, 22 Jun 2026 15:59:52 +0100 Subject: [PATCH 22/27] Enhance SQLServerDatatypeImporter to support BLOB type handling for varbinary columns --- .../sqlserver/in/SQLServerDatatypeImporter.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java b/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java index 400746220..0efb41e01 100644 --- a/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java +++ b/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java @@ -34,6 +34,19 @@ protected Type getBinaryType(String typeName, int columnSize, int decimalDigits, if (typeName.contains("timestamp") || typeName.contains("rowversion")) { return super.getBinaryType(typeName, 8, decimalDigits, numPrecRadix); } + return super.getBinaryType(typeName, columnSize, decimalDigits, numPrecRadix); } + + @Override + protected Type getVarbinaryType(String typeName, int columnSize, int decimalDigits, int numPrecRadix) { + if (typeName.equalsIgnoreCase("varbinary") && (columnSize == 2147483647 || columnSize == -1)) { + Type type = new SimpleTypeBinary(columnSize); + type.setSql99TypeName("BINARY LARGE OBJECT"); + type.setSql2008TypeName("BINARY LARGE OBJECT"); + return type; + } + + return super.getVarbinaryType(typeName, columnSize, decimalDigits, numPrecRadix); + } } From b5d4d307293b124f906e5ba67a40b022582a263c Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Thu, 25 Jun 2026 10:07:46 +0100 Subject: [PATCH 23/27] Add ExportModuleContextManager for managing export module context and enhance DatabaseMigration and ExternalLOBSFilter for improved type handling --- .../DatabaseMigration.java | 107 ++++++++++-------- .../managers/ExportModuleContextManager.java | 43 +++++++ .../externalLobs/ExternalLOBSFilter.java | 24 +++- 3 files changed, 119 insertions(+), 55 deletions(-) create mode 100644 dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java diff --git a/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java b/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java index 89548ca6f..11ac617e7 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java +++ b/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java @@ -15,6 +15,7 @@ import org.apache.commons.lang3.tuple.Pair; +import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.modules.DatabaseImportModule; import com.databasepreservation.model.modules.DatabaseModuleFactory; @@ -60,67 +61,75 @@ public DatabaseImportModule getImportModule() throws ModuleException { public void migrate() throws ModuleException { validate(); - // get import module and export module instance - Map importParameters = buildParametersFromStringParameters(importModuleFactory, - importModuleFactoryStringParameters); - Map exportParameters = buildParametersFromStringParameters(exportModuleFactory, - exportModuleFactoryStringParameters); - - DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter); - DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter); - - List beforeFilterModules = new ArrayList<>(); - List afterFilterModules = new ArrayList<>(); - for (int i = 0; i < filterFactories.size(); i++) { - Map filterParameters = new HashMap<>(); - if (!filterFactoriesStringParameters.isEmpty()) { - filterParameters = buildParametersFromStringParameters(filterFactories.get(i), - filterFactoriesStringParameters.get(i)); + try { + if (exportModuleFactory != null) { + ExportModuleContextManager.getInstance().setup(exportModuleFactory); } - if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) { - afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); - } else { - beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + // get import module and export module instance + Map importParameters = buildParametersFromStringParameters(importModuleFactory, + importModuleFactoryStringParameters); + Map exportParameters = buildParametersFromStringParameters(exportModuleFactory, + exportModuleFactoryStringParameters); + + DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter); + DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter); + + List beforeFilterModules = new ArrayList<>(); + List afterFilterModules = new ArrayList<>(); + for (int i = 0; i < filterFactories.size(); i++) { + Map filterParameters = new HashMap<>(); + if (!filterFactoriesStringParameters.isEmpty()) { + filterParameters = buildParametersFromStringParameters(filterFactories.get(i), + filterFactoriesStringParameters.get(i)); + } + + if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) { + afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + } else { + beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + } } - } - // set reporters - importModule.setOnceReporter(reporter); - for (DatabaseFilterModule filterModule : beforeFilterModules) { - filterModule.setOnceReporter(reporter); - } + // set reporters + importModule.setOnceReporter(reporter); + for (DatabaseFilterModule filterModule : beforeFilterModules) { + filterModule.setOnceReporter(reporter); + } - for (DatabaseFilterModule filterModule : afterFilterModules) { - filterModule.setOnceReporter(reporter); - } - for (DatabaseFilterModule filterModule : filterModules) { - filterModule.setOnceReporter(reporter); - } - exportModule.setOnceReporter(reporter); + for (DatabaseFilterModule filterModule : afterFilterModules) { + filterModule.setOnceReporter(reporter); + } + for (DatabaseFilterModule filterModule : filterModules) { + filterModule.setOnceReporter(reporter); + } + exportModule.setOnceReporter(reporter); - // create module chain with filters in the middle - Collections.reverse(filterModules); - Collections.reverse(beforeFilterModules); - Collections.reverse(afterFilterModules); + // create module chain with filters in the middle + Collections.reverse(filterModules); + Collections.reverse(beforeFilterModules); + Collections.reverse(afterFilterModules); - DatabaseFilterModule sinkModule = new SinkModule(); + DatabaseFilterModule sinkModule = new SinkModule(); - for (DatabaseFilterModule filterModule : afterFilterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : afterFilterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - sinkModule = exportModule.migrateDatabaseTo(sinkModule); + sinkModule = exportModule.migrateDatabaseTo(sinkModule); - for (DatabaseFilterModule filterModule : beforeFilterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : beforeFilterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - for (DatabaseFilterModule filterModule : filterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : filterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - importModule.migrateDatabaseTo(sinkModule); + importModule.migrateDatabaseTo(sinkModule); + } finally { + ExportModuleContextManager.destroy(); + } } /** diff --git a/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java b/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java new file mode 100644 index 000000000..ffe7da3ef --- /dev/null +++ b/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java @@ -0,0 +1,43 @@ +package com.databasepreservation.managers; + +import com.databasepreservation.model.exception.UnsupportedModuleException; +import com.databasepreservation.model.modules.DatabaseModuleFactory; +import com.databasepreservation.model.parameters.Parameters; + +/** + * @author Gabriel Barros + */ +public class ExportModuleContextManager { + private static ExportModuleContextManager instance = null; + private String moduleName; + private Parameters exportModuleParameters; + + public static ExportModuleContextManager getInstance() { + if (instance == null) { + instance = new ExportModuleContextManager(); + } + + return instance; + } + + public static void destroy() { + instance = null; + } + + public void setup(DatabaseModuleFactory exportModuleFactory) throws UnsupportedModuleException { + moduleName = exportModuleFactory.getModuleName(); + exportModuleParameters = exportModuleFactory.getExportModuleParameters(); + } + + public String getModuleName() { + return moduleName; + } + + public Parameters getExportModuleParameters() { + return exportModuleParameters; + } + + public boolean isSiadDKModule() { + return moduleName.contains("siard-dk"); + } +} diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java index bd34df3ab..50a99a33b 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java @@ -14,9 +14,11 @@ import java.util.Map; import java.util.Set; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.managers.ModuleConfigurationManager; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.NullCell; @@ -93,12 +95,7 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException Type original = column.getType(); description.append(". Original description: '").append(original.getDescription()).append("')"); - SimpleTypeBinary newType = new SimpleTypeBinary(); - newType.setSql99TypeName("BINARY VARYING", 1); - newType.setSql2008TypeName("BINARY VARYING", 1); - newType.setOriginalTypeName(original.getOriginalTypeName()); - newType.setOutsideDatabase(true); - + SimpleTypeBinary newType = getSimpleTypeBinary(original); column.setType(newType); column.setDescription(description.toString()); } @@ -110,6 +107,21 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException this.exportModule.handleStructure(structure); } + @NotNull + private static SimpleTypeBinary getSimpleTypeBinary(Type original) { + SimpleTypeBinary newType = new SimpleTypeBinary(); + if (ExportModuleContextManager.getInstance().isSiadDKModule()) { + newType.setSql99TypeName("BINARY LARGE OBJECT"); + newType.setSql2008TypeName("BINARY LARGE OBJECT"); + } else { + newType.setSql99TypeName("BINARY VARYING", 1); + newType.setSql2008TypeName("BINARY VARYING", 1); + } + newType.setOriginalTypeName(original.getOriginalTypeName()); + newType.setOutsideDatabase(true); + return newType; + } + @Override public void handleDataOpenSchema(String schemaName) throws ModuleException { this.exportModule.handleDataOpenSchema(schemaName); From dc309686f464af7f375afcda079144f69d4d4418 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Wed, 8 Jul 2026 14:11:29 +0100 Subject: [PATCH 24/27] refactor: cell value strip trailing --- .../ExternalLOBSCellHandlerFileSystem.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java index 162692682..2c59e4474 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java @@ -40,7 +40,9 @@ public Cell handleCell(String cellId, String cellValue) throws ModuleException { return newCell; } - Path blobPath = basePath.resolve(cellValue); + String cellValueStripTrailing = cellValue.stripTrailing(); + + Path blobPath = basePath.resolve(cellValueStripTrailing); if (Files.exists(blobPath)) { if (Files.isRegularFile(blobPath)) { @@ -48,15 +50,15 @@ public Cell handleCell(String cellId, String cellValue) throws ModuleException { newCell = new BinaryCell(cellId, new PathInputStreamProvider(blobPath)); } catch (ModuleException e) { reporter.ignored("Cell " + cellId, - blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValue); + blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing); } } else { reporter.ignored("Cell " + cellId, - blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValue); + blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing); } } else { reporter.ignored("Cell " + cellId, "Path: " + blobPath.toString() + " could not be found; Base path: " - + this.basePath + " Cell Value: " + cellValue); + + this.basePath + " Cell Value: " + cellValueStripTrailing); } return newCell; } From c364133d19b3869288176721bafbd2c79fa56f37 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Thu, 30 Jul 2026 12:09:40 +0100 Subject: [PATCH 25/27] fix: original filename of external lobs on doc indexes --- .../com/databasepreservation/model/data/BinaryCell.java | 4 ++++ .../modules/externalLobs/ExternalLOBSFilter.java | 4 ++++ .../siard/out/content/SIARDDKContentExportStrategy.java | 7 +++++++ .../siard/out/output/SIARDDKDatabaseExportModule.java | 3 ++- .../services/conversion/model/report/ConversionReport.java | 5 +++++ 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java index 63f8fbf3b..a5a028314 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java @@ -139,4 +139,8 @@ public long getLength() { public String getMimeType() { return mimeType; } + + public void setFile(String file) { + this.file = file; + } } diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java index 50a99a33b..b64d47e6a 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java @@ -20,6 +20,7 @@ import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.managers.ModuleConfigurationManager; +import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.NullCell; import com.databasepreservation.model.data.Row; @@ -167,6 +168,9 @@ public void handleDataRow(Row row) throws ModuleException { .get(currentTable.getId() + index); Cell newCell = getExternalLOBSCellHandler(externalLobsConfiguration).handleCell(cell.getId(), simpleCell.getSimpleData()); + if (newCell instanceof BinaryCell binaryCell) { + binaryCell.setFile(simpleCell.getSimpleData()); + } rowCells.set(index, newCell); } else { reporter.ignored("Cell " + cell.getId(), "reference to external LOB is null"); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index d5f227166..0a497a3ce 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -20,6 +20,7 @@ import java.util.zip.ZipInputStream; import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.jdom2.Document; import org.jdom2.Element; @@ -367,6 +368,12 @@ private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, in throw new ModuleException().withMessage("Missing conversion_report.json in cell archive."); } + String fileFromCell = binaryCell.getFile(); + if (fileFromCell != null) { + String filename = FilenameUtils.getName(fileFromCell).stripTrailing(); + report = report.withOriginalFilename(filename); + } + List siardPhysicalPaths = new ArrayList<>(); int fileCount = 0; String processedFilesExtension = "tif"; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 9fd917c1e..d750ca8c2 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -360,8 +360,9 @@ private ProcessedRowContext processRowAsync(Row row) throws ModuleException { // TODO: Handle multiple files per cell if needed. Currently assumes single file // output. - Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.zipFile()), + BinaryCell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.zipFile()), "application/zip"); + newCell.setFile(binCell.getFile()); cells.set(i, newCell); // Track extracted parts to clean them individually later diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java index 4deafef2c..32aaec45f 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java @@ -19,4 +19,9 @@ public ConversionReport withContext(DbptkContext context) { return new ConversionReport(jobId, status, originalFilename, totalArtifactsProduced, artifacts, errorMessage, context); } + + public ConversionReport withOriginalFilename(String newFilename) { + return new ConversionReport(jobId, status, newFilename, totalArtifactsProduced, artifacts, errorMessage, + dbptkContext); + } } From c8a5096abc67d27e479fe6ef6802102b06820471 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Wed, 9 Sep 2026 13:54:21 +0100 Subject: [PATCH 26/27] fix: issues after rebase --- .../content/SIARDDKContentExportStrategy.java | 39 ++++++++++++++++++- .../SIARDDK1007DatabaseExportModule.java | 5 ++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 0a497a3ce..136e945e5 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -40,6 +40,7 @@ import com.databasepreservation.model.structure.ColumnStructure; import com.databasepreservation.model.structure.SchemaStructure; import com.databasepreservation.model.structure.TableStructure; +import com.databasepreservation.modules.siard.common.LargeObject; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDConstants; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; @@ -343,7 +344,9 @@ public Row tableRow(Row row) throws ModuleException { logger.warn( "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", mimeType, tableCounter, columnIndex); - whiteNilCell(columnIndex); + //whiteNilCell(columnIndex); + + archiveRawLob(binaryCell, columnIndex); } } else { // never happens @@ -421,6 +424,40 @@ private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, in } } + private void archiveRawLob(BinaryCell binaryCell, int columnIndex) throws ModuleException, IOException { + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; + String fileExtension; + if (mimetypeHandler.isMimetypeAllowed(mimeType)) { + fileExtension = mimetypeHandler.getFileExtension(mimeType); + } else { + fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; + foundUnknownMimetype = true; + } + + double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); + + String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + "1." + fileExtension; + LargeObject blob = new LargeObject(binaryCell, path); + + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); + InputStream in = blob.getInputStreamProvider().createInputStream(); + IOUtils.copy(in, out); + IOUtils.closeQuietly(in); + IOUtils.closeQuietly(out); + blob.getInputStreamProvider().cleanResources(); + + lobsTracker.addLOB(lobSizeMB); + + writeLobReferenceToXml(columnIndex); + + String originalFilename = binaryCell.getFile() != null ? FilenameUtils.getName(binaryCell.getFile()).stripTrailing() + : "originalFilename"; + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + originalFilename, fileExtension, null); + + SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + } + private ConversionReport extractReportFromZip(BinaryCell binaryCell) throws Exception { try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { ZipEntry zipEntry; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java index e775df706..cac3d926d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java @@ -7,9 +7,10 @@ */ package com.databasepreservation.modules.siard.out.output; -import com.databasepreservation.modules.siard.bindings.siard_dk_1007.SiardDiark; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import dk.sa.xmlns.diark._1_0.fileindex.FileIndexType; + /** * @author António Lindo * @@ -27,6 +28,6 @@ String getJAXBContext() { @Override Class getJAXBContextClass() { - return SiardDiark.class; + return FileIndexType.class; } } From e39b143f7bdbb4385b5ee58947f4329c56594b40 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Tue, 15 Sep 2026 13:17:24 +0100 Subject: [PATCH 27/27] refactor: non normalizer blobs and siard tests --- .../testing/integration/siard/SiardTest.java | 7 +- .../content/SIARDDKContentExportStrategy.java | 78 +++++++++---------- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java index 42a7f4db0..33f62813b 100644 --- a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java +++ b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java @@ -36,6 +36,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import com.databasepreservation.common.io.providers.TemporaryPathInputStreamProvider; import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.Row; @@ -539,15 +540,15 @@ protected DatabaseStructure generateDatabaseStructure() throws ModuleException, new Row(1, Arrays.asList(new SimpleCell("table02.col121.0", "1"), new SimpleCell("table02.col122.0", "3"), new SimpleCell("table02.col123.0", "abc"), new SimpleCell("table02.col124.0", "def"), - new BinaryCell("table02.col125.0", newBlob()))), + new BinaryCell("table02.col125.0", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))), new Row(2, Arrays.asList(new SimpleCell("table02.col121.1", "2"), new SimpleCell("table02.col122.1", "1"), new SimpleCell("table02.col123.1", "dns"), new SimpleCell("table02.col124.1", "dud"), - new BinaryCell("table02.col125.1", newBlob()))), + new BinaryCell("table02.col125.1", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))), new Row(3, Arrays.asList(new SimpleCell("table02.col121.2", "3"), new SimpleCell("table02.col122.2", "2"), new SimpleCell("table02.col123.2", "usl"), new SimpleCell("table02.col124.2", "aps"), - new BinaryCell("table02.col125.2", newBlob()))))); + new BinaryCell("table02.col125.2", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))))); tableRows.put("schema02.table01", new ArrayList()); tableRows.put("schema02.table02", new ArrayList()); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 136e945e5..fdb6e9866 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -341,12 +341,7 @@ public Row tableRow(Row row) throws ModuleException { if (mimeType.equals("application/zip")) { processConvertedLobArchive(binaryCell, row.getIndex(), columnIndex); } else { - logger.warn( - "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", - mimeType, tableCounter, columnIndex); - //whiteNilCell(columnIndex); - - archiveRawLob(binaryCell, columnIndex); + processRawLobFile(binaryCell, columnIndex); } } else { // never happens @@ -363,6 +358,43 @@ public Row tableRow(Row row) throws ModuleException { return row; } + private void processRawLobFile(BinaryCell binaryCell, int columnIndex) throws ModuleException, IOException { + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; + String fileExtension; + if (mimetypeHandler.isMimetypeAllowed(mimeType)) { + fileExtension = mimetypeHandler.getFileExtension(mimeType); + } else { + logger.warn( + "Found BLOB with unsupported mimetype '{}' in table {}, column {}. archiving as .bin file.", + mimeType, tableCounter, columnIndex); + fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; + foundUnknownMimetype = true; + } + + double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); + + String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + "1." + fileExtension; + LargeObject blob = new LargeObject(binaryCell, path); + + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); + InputStream in = blob.getInputStreamProvider().createInputStream(); + IOUtils.copy(in, out); + IOUtils.closeQuietly(in); + IOUtils.closeQuietly(out); + blob.getInputStreamProvider().cleanResources(); + + lobsTracker.addLOB(lobSizeMB); + + writeLobReferenceToXml(columnIndex); + + String originalFilename = binaryCell.getFile() != null ? FilenameUtils.getName(binaryCell.getFile()).stripTrailing() + : "originalFilename"; + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + originalFilename, fileExtension, null); + + SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + } + private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, int columnIndex) throws ModuleException { try { @@ -424,40 +456,6 @@ private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, in } } - private void archiveRawLob(BinaryCell binaryCell, int columnIndex) throws ModuleException, IOException { - String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; - String fileExtension; - if (mimetypeHandler.isMimetypeAllowed(mimeType)) { - fileExtension = mimetypeHandler.getFileExtension(mimeType); - } else { - fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; - foundUnknownMimetype = true; - } - - double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); - - String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + "1." + fileExtension; - LargeObject blob = new LargeObject(binaryCell, path); - - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); - InputStream in = blob.getInputStreamProvider().createInputStream(); - IOUtils.copy(in, out); - IOUtils.closeQuietly(in); - IOUtils.closeQuietly(out); - blob.getInputStreamProvider().cleanResources(); - - lobsTracker.addLOB(lobSizeMB); - - writeLobReferenceToXml(columnIndex); - - String originalFilename = binaryCell.getFile() != null ? FilenameUtils.getName(binaryCell.getFile()).stripTrailing() - : "originalFilename"; - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - originalFilename, fileExtension, null); - - SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); - } - private ConversionReport extractReportFromZip(BinaryCell binaryCell) throws Exception { try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { ZipEntry zipEntry;