diff --git a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java index 42a7f4db0..33f62813b 100644 --- a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java +++ b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java @@ -36,6 +36,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import com.databasepreservation.common.io.providers.TemporaryPathInputStreamProvider; import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.Row; @@ -539,15 +540,15 @@ protected DatabaseStructure generateDatabaseStructure() throws ModuleException, new Row(1, Arrays.asList(new SimpleCell("table02.col121.0", "1"), new SimpleCell("table02.col122.0", "3"), new SimpleCell("table02.col123.0", "abc"), new SimpleCell("table02.col124.0", "def"), - new BinaryCell("table02.col125.0", newBlob()))), + new BinaryCell("table02.col125.0", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))), new Row(2, Arrays.asList(new SimpleCell("table02.col121.1", "2"), new SimpleCell("table02.col122.1", "1"), new SimpleCell("table02.col123.1", "dns"), new SimpleCell("table02.col124.1", "dud"), - new BinaryCell("table02.col125.1", newBlob()))), + new BinaryCell("table02.col125.1", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))), new Row(3, Arrays.asList(new SimpleCell("table02.col121.2", "3"), new SimpleCell("table02.col122.2", "2"), new SimpleCell("table02.col123.2", "usl"), new SimpleCell("table02.col124.2", "aps"), - new BinaryCell("table02.col125.2", newBlob()))))); + new BinaryCell("table02.col125.2", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))))); tableRows.put("schema02.table01", new ArrayList()); tableRows.put("schema02.table02", new ArrayList()); diff --git a/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java b/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java index 89548ca6f..11ac617e7 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java +++ b/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java @@ -15,6 +15,7 @@ import org.apache.commons.lang3.tuple.Pair; +import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.modules.DatabaseImportModule; import com.databasepreservation.model.modules.DatabaseModuleFactory; @@ -60,67 +61,75 @@ public DatabaseImportModule getImportModule() throws ModuleException { public void migrate() throws ModuleException { validate(); - // get import module and export module instance - Map importParameters = buildParametersFromStringParameters(importModuleFactory, - importModuleFactoryStringParameters); - Map exportParameters = buildParametersFromStringParameters(exportModuleFactory, - exportModuleFactoryStringParameters); - - DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter); - DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter); - - List beforeFilterModules = new ArrayList<>(); - List afterFilterModules = new ArrayList<>(); - for (int i = 0; i < filterFactories.size(); i++) { - Map filterParameters = new HashMap<>(); - if (!filterFactoriesStringParameters.isEmpty()) { - filterParameters = buildParametersFromStringParameters(filterFactories.get(i), - filterFactoriesStringParameters.get(i)); + try { + if (exportModuleFactory != null) { + ExportModuleContextManager.getInstance().setup(exportModuleFactory); } - if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) { - afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); - } else { - beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + // get import module and export module instance + Map importParameters = buildParametersFromStringParameters(importModuleFactory, + importModuleFactoryStringParameters); + Map exportParameters = buildParametersFromStringParameters(exportModuleFactory, + exportModuleFactoryStringParameters); + + DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter); + DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter); + + List beforeFilterModules = new ArrayList<>(); + List afterFilterModules = new ArrayList<>(); + for (int i = 0; i < filterFactories.size(); i++) { + Map filterParameters = new HashMap<>(); + if (!filterFactoriesStringParameters.isEmpty()) { + filterParameters = buildParametersFromStringParameters(filterFactories.get(i), + filterFactoriesStringParameters.get(i)); + } + + if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) { + afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + } else { + beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + } } - } - // set reporters - importModule.setOnceReporter(reporter); - for (DatabaseFilterModule filterModule : beforeFilterModules) { - filterModule.setOnceReporter(reporter); - } + // set reporters + importModule.setOnceReporter(reporter); + for (DatabaseFilterModule filterModule : beforeFilterModules) { + filterModule.setOnceReporter(reporter); + } - for (DatabaseFilterModule filterModule : afterFilterModules) { - filterModule.setOnceReporter(reporter); - } - for (DatabaseFilterModule filterModule : filterModules) { - filterModule.setOnceReporter(reporter); - } - exportModule.setOnceReporter(reporter); + for (DatabaseFilterModule filterModule : afterFilterModules) { + filterModule.setOnceReporter(reporter); + } + for (DatabaseFilterModule filterModule : filterModules) { + filterModule.setOnceReporter(reporter); + } + exportModule.setOnceReporter(reporter); - // create module chain with filters in the middle - Collections.reverse(filterModules); - Collections.reverse(beforeFilterModules); - Collections.reverse(afterFilterModules); + // create module chain with filters in the middle + Collections.reverse(filterModules); + Collections.reverse(beforeFilterModules); + Collections.reverse(afterFilterModules); - DatabaseFilterModule sinkModule = new SinkModule(); + DatabaseFilterModule sinkModule = new SinkModule(); - for (DatabaseFilterModule filterModule : afterFilterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : afterFilterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - sinkModule = exportModule.migrateDatabaseTo(sinkModule); + sinkModule = exportModule.migrateDatabaseTo(sinkModule); - for (DatabaseFilterModule filterModule : beforeFilterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : beforeFilterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - for (DatabaseFilterModule filterModule : filterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : filterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - importModule.migrateDatabaseTo(sinkModule); + importModule.migrateDatabaseTo(sinkModule); + } finally { + ExportModuleContextManager.destroy(); + } } /** diff --git a/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java b/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java new file mode 100644 index 000000000..ffe7da3ef --- /dev/null +++ b/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java @@ -0,0 +1,43 @@ +package com.databasepreservation.managers; + +import com.databasepreservation.model.exception.UnsupportedModuleException; +import com.databasepreservation.model.modules.DatabaseModuleFactory; +import com.databasepreservation.model.parameters.Parameters; + +/** + * @author Gabriel Barros + */ +public class ExportModuleContextManager { + private static ExportModuleContextManager instance = null; + private String moduleName; + private Parameters exportModuleParameters; + + public static ExportModuleContextManager getInstance() { + if (instance == null) { + instance = new ExportModuleContextManager(); + } + + return instance; + } + + public static void destroy() { + instance = null; + } + + public void setup(DatabaseModuleFactory exportModuleFactory) throws UnsupportedModuleException { + moduleName = exportModuleFactory.getModuleName(); + exportModuleParameters = exportModuleFactory.getExportModuleParameters(); + } + + public String getModuleName() { + return moduleName; + } + + public Parameters getExportModuleParameters() { + return exportModuleParameters; + } + + public boolean isSiadDKModule() { + return moduleName.contains("siard-dk"); + } +} diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java index 8b9149f11..a5a028314 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java @@ -28,6 +28,7 @@ public class BinaryCell extends Cell implements InputStreamProvider { private InputStreamProvider inputStreamProvider; private String file; private long length; + private String mimeType; /** * Creates a binary cell. This binary cell will mostly just be a wrapper around @@ -75,6 +76,12 @@ public BinaryCell(String id, InputStreamProvider inputStreamProvider) { this.inputStreamProvider = inputStreamProvider; } + public BinaryCell(String id, InputStreamProvider inputStreamProvider, String mimeType) { + super(id); + this.inputStreamProvider = inputStreamProvider; + this.mimeType = mimeType; + } + /** * Creates a binary cell. This binary cell is a wrapper around a * ProvidesInputStream object (whilst also providing Cell functionality). @@ -128,4 +135,12 @@ public String getFile() { public long getLength() { return length; } + + public String getMimeType() { + return mimeType; + } + + public void setFile(String file) { + this.file = file; + } } diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java b/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java index 03f74e006..378250821 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java @@ -28,7 +28,7 @@ public enum INPUT_TYPE { /* GUI Helper for SIARD Export Module */ public enum CATEGORY_TYPE { - SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, NONE + SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, CONVERSION_SERVICE_OPTIONS, NONE } public enum FILE_FILTER_TYPE { @@ -312,7 +312,9 @@ public Parameter defaultSelectedIndex(Integer index) { return this; } - public Integer getDefaultSelectedIndex() { return defaultSelectedIndex; } + public Integer getDefaultSelectedIndex() { + return defaultSelectedIndex; + } /** * Gets the export option type for this parameter; Helper to automatize the diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java index 162692682..2c59e4474 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java @@ -40,7 +40,9 @@ public Cell handleCell(String cellId, String cellValue) throws ModuleException { return newCell; } - Path blobPath = basePath.resolve(cellValue); + String cellValueStripTrailing = cellValue.stripTrailing(); + + Path blobPath = basePath.resolve(cellValueStripTrailing); if (Files.exists(blobPath)) { if (Files.isRegularFile(blobPath)) { @@ -48,15 +50,15 @@ public Cell handleCell(String cellId, String cellValue) throws ModuleException { newCell = new BinaryCell(cellId, new PathInputStreamProvider(blobPath)); } catch (ModuleException e) { reporter.ignored("Cell " + cellId, - blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValue); + blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing); } } else { reporter.ignored("Cell " + cellId, - blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValue); + blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing); } } else { reporter.ignored("Cell " + cellId, "Path: " + blobPath.toString() + " could not be found; Base path: " - + this.basePath + " Cell Value: " + cellValue); + + this.basePath + " Cell Value: " + cellValueStripTrailing); } return newCell; } diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java index bd34df3ab..b64d47e6a 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java @@ -14,10 +14,13 @@ import java.util.Map; import java.util.Set; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.managers.ModuleConfigurationManager; +import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.NullCell; import com.databasepreservation.model.data.Row; @@ -93,12 +96,7 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException Type original = column.getType(); description.append(". Original description: '").append(original.getDescription()).append("')"); - SimpleTypeBinary newType = new SimpleTypeBinary(); - newType.setSql99TypeName("BINARY VARYING", 1); - newType.setSql2008TypeName("BINARY VARYING", 1); - newType.setOriginalTypeName(original.getOriginalTypeName()); - newType.setOutsideDatabase(true); - + SimpleTypeBinary newType = getSimpleTypeBinary(original); column.setType(newType); column.setDescription(description.toString()); } @@ -110,6 +108,21 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException this.exportModule.handleStructure(structure); } + @NotNull + private static SimpleTypeBinary getSimpleTypeBinary(Type original) { + SimpleTypeBinary newType = new SimpleTypeBinary(); + if (ExportModuleContextManager.getInstance().isSiadDKModule()) { + newType.setSql99TypeName("BINARY LARGE OBJECT"); + newType.setSql2008TypeName("BINARY LARGE OBJECT"); + } else { + newType.setSql99TypeName("BINARY VARYING", 1); + newType.setSql2008TypeName("BINARY VARYING", 1); + } + newType.setOriginalTypeName(original.getOriginalTypeName()); + newType.setOutsideDatabase(true); + return newType; + } + @Override public void handleDataOpenSchema(String schemaName) throws ModuleException { this.exportModule.handleDataOpenSchema(schemaName); @@ -155,6 +168,9 @@ public void handleDataRow(Row row) throws ModuleException { .get(currentTable.getId() + index); Cell newCell = getExternalLOBSCellHandler(externalLobsConfiguration).handleCell(cell.getId(), simpleCell.getSimpleData()); + if (newCell instanceof BinaryCell binaryCell) { + binaryCell.setFile(simpleCell.getSimpleData()); + } rowCells.set(index, newCell); } else { reporter.ignored("Cell " + cell.getId(), "reference to external LOB is null"); diff --git a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java index 9d029d73c..92a7b009e 100644 --- a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java +++ b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java @@ -684,7 +684,7 @@ protected void handleSimpleTypeStringDataCell(String data, PreparedStatement ps, protected void handleSimpleTypeNumericExactDataCell(String data, PreparedStatement ps, int index, Cell cell, ColumnStructure column) throws SQLException { - if (data != null) { + if (data != null && !data.isEmpty()) { BigDecimal bd = new BigDecimal(data); ps.setBigDecimal(index, bd); } else { diff --git a/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java b/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java index 7d63862a6..de1449ad1 100644 --- a/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java +++ b/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java @@ -59,7 +59,7 @@ public String escapeSchemaName(String schema) { @Override public String escapeTableName(String table) { - return getStartQuote() + table + getEndQuote(); + return getStartQuote() + StringUtils.strip(table, "\"") + getEndQuote(); } /** diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java index f1bc0850a..87e038dcd 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java @@ -42,6 +42,9 @@ public abstract class SIARDDKModuleFactory implements DatabaseModuleFactory { public static final String PARAMETER_AS_SCHEMA = "as-schema"; public static final String PARAMETER_LOBS_PER_FOLDER = "lobs-per-folder"; public static final String PARAMETER_LOBS_FOLDER_SIZE = "lobs-folder-size"; + public static final String PARAMETER_LOB_CONVERSION_ENABLED = "lob-conversion"; + public static final String PARAMETER_LOB_CONVERSION_ENDPOINT = "lob-conversion-endpoint"; + public static final String PARAMETER_LOB_CONVERSION_TARGET_FORMAT = "lob-conversion-target-format"; // TODO: As things are now, are we not always generating the '.1' version of // the archive (indicating that the last .[1-9][0-9] should perhaps not be @@ -75,6 +78,20 @@ public abstract class SIARDDKModuleFactory implements DatabaseModuleFactory { .description("The maximum size (in megabytes) of the docCollection folders (default is 1000 MB").required(false) .hasArgument(true).setOptionalArgument(false).valueIfNotSet("1000"); + private static final Parameter lobConversionEnabled = new Parameter().shortName("lc") + .longName(PARAMETER_LOB_CONVERSION_ENABLED).description("Enables asynchronous LOB conversion via HTTP.") + .hasArgument(false).setOptionalArgument(false).required(false).valueIfSet("true").valueIfNotSet("false"); + + private static final Parameter lobConversionEndpoint = new Parameter().shortName("lce") + .longName(PARAMETER_LOB_CONVERSION_ENDPOINT) + .description("The API endpoint URL for the LOB conversion service (default is http://localhost:8087).") + .hasArgument(true).setOptionalArgument(false).required(false).valueIfNotSet("http://localhost:8087"); + + private static final Parameter lobConversionTargetFormat = new Parameter().shortName("lcf") + .longName(PARAMETER_LOB_CONVERSION_TARGET_FORMAT) + .description("Target MIME type format for the LOB conversion (default is image/tiff).").hasArgument(true) + .setOptionalArgument(false).required(false).valueIfNotSet("image/tiff"); + // This is not used now, but will be used later // private static final Parameter clobType = new // Parameter().shortName("ct").longName("clobtype") @@ -120,6 +137,9 @@ public Map getAllParameters() { parameterMap.put(importAsSchema.longName(), importAsSchema); parameterMap.put(lobsPerFolder.longName(), lobsPerFolder); parameterMap.put(lobsFolderSize.longName(), lobsFolderSize); + parameterMap.put(lobConversionEnabled.longName(), lobConversionEnabled); + parameterMap.put(lobConversionEndpoint.longName(), lobConversionEndpoint); + parameterMap.put(lobConversionTargetFormat.longName(), lobConversionTargetFormat); // to be used later... // parameterMap.put(clobType.longName(), clobType); // parameterMap.put(clobLength.longName(), clobLength); @@ -145,18 +165,22 @@ public Parameters getExportModuleParameters() throws UnsupportedModuleException // contextDocumentationIndex, contextDocmentationFolder, // clobType, clobLength), null); - return new Parameters( - Arrays.asList( - folder.inputType(Parameter.INPUT_TYPE.FOLDER).exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - archiveIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN).fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - contextDocumentationIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN) - .fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - contextDocumentationFolder.inputType(Parameter.INPUT_TYPE.FOLDER) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), - lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS)), + return new Parameters(Arrays.asList( + folder.inputType(Parameter.INPUT_TYPE.FOLDER).exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + archiveIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN).fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + contextDocumentationIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN) + .fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + contextDocumentationFolder.inputType(Parameter.INPUT_TYPE.FOLDER) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionEnabled.inputType(Parameter.INPUT_TYPE.CHECKBOX).exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS), + lobConversionEndpoint.inputType(Parameter.INPUT_TYPE.TEXT) + .exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS), + lobConversionTargetFormat.inputType(Parameter.INPUT_TYPE.TEXT) + .exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS)), null); } @@ -193,6 +217,21 @@ public DatabaseFilterModule buildExportModule(Map parameters, pLobsFolderSize = parameters.get(lobsFolderSize); } + String pLobConversionEnabled = lobConversionEnabled.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionEnabled))) { + pLobConversionEnabled = parameters.get(lobConversionEnabled); + } + + String pLobConversionEndpoint = lobConversionEndpoint.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionEndpoint))) { + pLobConversionEndpoint = parameters.get(lobConversionEndpoint); + } + + String pLobConversionTargetFormat = lobConversionTargetFormat.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionTargetFormat))) { + pLobConversionTargetFormat = parameters.get(lobConversionTargetFormat); + } + // to be used later... // String pClobType = parameters.get(clobType); // String pClobLength = parameters.get(clobLength); @@ -205,6 +244,9 @@ public DatabaseFilterModule buildExportModule(Map parameters, exportModuleArgs.put(contextDocumentationFolder.longName(), pContextDocumentationFolder); exportModuleArgs.put(lobsPerFolder.longName(), pLobsPerFolder); exportModuleArgs.put(lobsFolderSize.longName(), pLobsFolderSize); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_ENABLED, pLobConversionEnabled); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_ENDPOINT, pLobConversionEndpoint); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_TARGET_FORMAT, pLobConversionTargetFormat); // to be used later... // exportModuleArgs.put(clobType.longName(), pClobType); @@ -227,6 +269,18 @@ public DatabaseFilterModule buildExportModule(Map parameters, exportModuleParameters.add(lobsFolderSize.longName()); exportModuleParameters.add(pLobsFolderSize); } + if (!pLobConversionEnabled.equals(lobConversionEnabled.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_ENABLED); + exportModuleParameters.add(pLobConversionEnabled); + } + if (!pLobConversionEndpoint.equals(lobConversionEndpoint.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_ENDPOINT); + exportModuleParameters.add(pLobConversionEndpoint); + } + if (!pLobConversionTargetFormat.equals(lobConversionTargetFormat.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_TARGET_FORMAT); + exportModuleParameters.add(pLobConversionTargetFormat); + } reporter.exportModuleParameters(getModuleName(), exportModuleParameters.toArray(new String[0])); return createSIARDDKExportModuleInstance(exportModuleArgs).getDatabaseExportModule(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java index a4fe44a1a..f0a445522 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java @@ -7,12 +7,12 @@ */ package com.databasepreservation.modules.siard.common.path; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; - import java.security.InvalidParameterException; import java.util.ArrayList; import java.util.Arrays; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; + /** * @author Andreas Kring * @@ -54,7 +54,8 @@ public boolean checkFilename(String filename) { // Valid filenames String[] validFileNames = {SIARDDKConstants.TABLE_INDEX, SIARDDKConstants.ARCHIVE_INDEX, SIARDDKConstants.DOC_INDEX, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, SIARDDKConstants.FILE_INDEX, - SIARDDKConstants.DOCUMENT_IDENTIFICATION, SIARDDKConstants.XML_SCHEMA, "fileIndex_original", "docIndex_original"}; + SIARDDKConstants.DOCUMENT_IDENTIFICATION, SIARDDKConstants.XML_SCHEMA, SIARDDKConstants.RESEARCH_INDEX, + "fileIndex_original", "docIndex_original"}; ArrayList validFilenames = new ArrayList(Arrays.asList(validFileNames)); if (validFilenames.contains(filename)) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java index 68dd9d90b..203433413 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java @@ -54,6 +54,7 @@ public class SIARDDKConstants { public static final String ARCHIVE_INDEX = "archiveIndex"; public static final String TABLE_INDEX = "tableIndex"; public static final String FILE_INDEX = "fileIndex"; + public static final String RESEARCH_INDEX = "researchIndex"; public static final String DOC_INDEX = "docIndex"; public static final String DOCUMENT_IDENTIFICATION = "documentIdentification"; public static final String XML_SCHEMA = "XMLSchema"; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java index c02344dd1..25bdeb1fe 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java @@ -323,7 +323,7 @@ private TableStructure createContextDocumentationTable() throws ModuleException virtualTable.setRows(contextDocumentationIndex.getDocument().size()); virtualTable.setColumns(createContextDocumentsTableColumns()); virtualTable.setPrimaryKey(createVirtualPrimaryKey( - SIARDDKConstants.CONTEXT_DOCUMENTATION_VIRTUAL_TABLE_PRIMARY_KEY_NAME, SIARDDKConstants.DID)); + SIARDDKConstants.CONTEXT_DOCUMENTATION_VIRTUAL_TABLE_PRIMARY_KEY_NAME, SIARDDKConstants.DOCUMENT_ID)); return virtualTable; } } catch (FileNotFoundException e) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 47163f158..fdb6e9866 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -7,16 +7,20 @@ */ package com.databasepreservation.modules.siard.out.content; -import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.jdom2.Document; import org.jdom2.Element; @@ -45,6 +49,11 @@ import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import com.databasepreservation.modules.siard.services.conversion.LobConversionAuditor; +import com.databasepreservation.modules.siard.services.conversion.model.report.ArtifactReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.DbptkContext; +import com.fasterxml.jackson.databind.ObjectMapper; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -69,6 +78,9 @@ public class SIARDDKContentExportStrategy implements ContentExportStrategy { private final LOBsTracker lobsTracker; private final MimetypeHandler mimetypeHandler; + private final LobConversionAuditor auditor; + private final ObjectMapper mapper; + private Reporter reporter; public SIARDDKContentExportStrategy(SIARDDKExportModule siarddkExportModule) { @@ -85,6 +97,11 @@ public SIARDDKContentExportStrategy(SIARDDKExportModule siarddkExportModule) { baseContainer = siarddkExportModule.getMainContainer(); writeStrategy = siarddkExportModule.getWriteStrategy(); lobsTracker = siarddkExportModule.getLobsTracker(); + + this.mapper = new ObjectMapper(); + Path exportRoot = baseContainer.getPath().getParent(); + String archiveName = baseContainer.getPath().getFileName().toString(); + this.auditor = new LobConversionAuditor(exportRoot, archiveName); } @Override @@ -287,16 +304,14 @@ public Row tableRow(Row row) throws ModuleException { binaryCell.cleanResources(); } } else { - tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + whiteNilCell(columnIndex); } } else { // cell must contain BLOB or CLOB if (cell instanceof NullCell) { - tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + whiteNilCell(columnIndex); } else if (cell instanceof SimpleCell) { // CLOB is not NULL @@ -317,76 +332,165 @@ public Row tableRow(Row row) throws ModuleException { } else if (cell instanceof BinaryCell) { // BLOB case - final BinaryCell binaryCell = (BinaryCell) cell; + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; - // BLOB is not NULL + // ------------------------------------------------------------- + // BLOB EXTRACTION DELEGATION + // ------------------------------------------------------------- + if (mimeType.equals("application/zip")) { + processConvertedLobArchive(binaryCell, row.getIndex(), columnIndex); + } else { + processRawLobFile(binaryCell, columnIndex); + } + } else { + // never happens + } + } + } - double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); - lobsTracker.addLOB(lobSizeMB); // Only if LOB not NULL + tableXmlWriter.append(TAB).append("\n"); - // Determine the mimetype (Tika should use an inputstream which - // supports marks) + } catch (IOException e) { + throw new ModuleException().withMessage("Could not write row " + row.toString()).withCause(e); + } - InputStream is = new BufferedInputStream(binaryCell.createInputStream()); - // Removed because TIKA was a security vulnerability and this feature was not - // needed/not fully implemented (see #341) - String mimeType = "unsupported"; - IOUtils.closeQuietly(is); + return row; + } - // Archive BLOB - simultaneous writing always supported for - // SIARDDK + private void processRawLobFile(BinaryCell binaryCell, int columnIndex) throws ModuleException, IOException { + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; + String fileExtension; + if (mimetypeHandler.isMimetypeAllowed(mimeType)) { + fileExtension = mimetypeHandler.getFileExtension(mimeType); + } else { + logger.warn( + "Found BLOB with unsupported mimetype '{}' in table {}, column {}. archiving as .bin file.", + mimeType, tableCounter, columnIndex); + fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; + foundUnknownMimetype = true; + } - tableXmlWriter.append(TAB).append(TAB).append("") - .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); + double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); - String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); - String fileExtension; - if (mimetypeHandler.isMimetypeAllowed(mimeType)) { - fileExtension = mimetypeHandler.getFileExtension(mimeType); - } else { - fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; - // Log (table level) that unknown BLOB mimetype was detected - foundUnknownMimetype = true; - } - path += fileExtension; + String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + "1." + fileExtension; + LargeObject blob = new LargeObject(binaryCell, path); - LargeObject blob = new LargeObject(binaryCell, path); + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); + InputStream in = blob.getInputStreamProvider().createInputStream(); + IOUtils.copy(in, out); + IOUtils.closeQuietly(in); + IOUtils.closeQuietly(out); + blob.getInputStreamProvider().cleanResources(); - // Create new FileIndexFileStrategy + lobsTracker.addLOB(lobSizeMB); - // Write the BLOB - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); - InputStream in = blob.getInputStreamProvider().createInputStream(); - IOUtils.copy(in, out); - IOUtils.closeQuietly(in); - IOUtils.closeQuietly(out); - blob.getInputStreamProvider().cleanResources(); + writeLobReferenceToXml(columnIndex); - // Add file to docIndex (a lot easier to do here even though we - // are dealing with metadata) + String originalFilename = binaryCell.getFile() != null ? FilenameUtils.getName(binaryCell.getFile()).stripTrailing() + : "originalFilename"; + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + originalFilename, fileExtension, null); - // TO-DO: obtain (how?) hardcoded values - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - "originalFilename", fileExtension, null); + SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + } - // Add file to fileIndex - SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, int columnIndex) + throws ModuleException { + try { + ConversionReport report = extractReportFromZip(binaryCell); + if (report == null) { + throw new ModuleException().withMessage("Missing conversion_report.json in cell archive."); + } - } else { - // never happens + String fileFromCell = binaryCell.getFile(); + if (fileFromCell != null) { + String filename = FilenameUtils.getName(fileFromCell).stripTrailing(); + report = report.withOriginalFilename(filename); + } + + List siardPhysicalPaths = new ArrayList<>(); + int fileCount = 0; + String processedFilesExtension = "tif"; + + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) + continue; + + ArtifactReport artifactMeta = findArtifactMetadata(report.artifacts(), zipEntry.getName()); + + if (artifactMeta == null || artifactMeta.isBypassed()) { + logger.warn("Ignoring bypassed or unknown file: {}. Reason: {}", zipEntry.getName(), + artifactMeta != null ? artifactMeta.errorMessage() : "Not in report"); + continue; } + + String fileExt = mimetypeHandler.getFileExtension(artifactMeta.finalMimeType()); + processedFilesExtension = fileExt; + fileCount++; + + String outputPath = writeLobToSiardStorage(zis, fileCount, fileExt); + siardPhysicalPaths.add(outputPath); } } - tableXmlWriter.append(TAB).append("\n"); + double lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); + lobsTracker.addLOB(lobSizeTotal); - } catch (IOException e) { - throw new ModuleException().withMessage("Could not write row " + row.toString()).withCause(e); + if (fileCount > 0) { + writeLobReferenceToXml(columnIndex); + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + report.originalFilename(), processedFilesExtension, null); + } else { + whiteNilCell(columnIndex); + } + + ConversionReport enrichedReport = report + .withContext(new DbptkContext(tableCounter, rowIndex, columnIndex, siardPhysicalPaths)); + auditor.appendAuditRecord(enrichedReport); + + } catch (Exception e) { + throw new ModuleException().withMessage("Failed to process converted ZIP archive").withCause(e); } + } - return row; + private ConversionReport extractReportFromZip(BinaryCell binaryCell) throws Exception { + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) { + return mapper.readValue(zis.readAllBytes(), ConversionReport.class); + } + } + } + return null; + } + + private ArtifactReport findArtifactMetadata(List artifacts, String fileName) { + if (artifacts == null) + return null; + return artifacts.stream().filter(a -> fileName.equals(a.logicalName())).findFirst().orElse(null); + } + + private String writeLobToSiardStorage(InputStream zis, int fileCount, String extension) throws Exception { + String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + fileCount + "." + extension; + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, writeStrategy); + zis.transferTo(out); + SIARDDKFileIndexFileStrategy.addFile(outputPath); + return outputPath; + } + + private void writeLobReferenceToXml(int columnIndex) throws IOException { + tableXmlWriter.append(TAB).append(TAB).append("") + .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); + } + + private void whiteNilCell(int columnIndex) throws IOException { + tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java index ae10da9ae..04720fbf4 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java @@ -12,6 +12,8 @@ import dk.sa.xmlns.diark._1_0.docindex.DocIndexType; import dk.sa.xmlns.diark._1_0.docindex.DocumentType; +import dk.sa.xmlns.diark._1_0.docindex.ObjectFactory; +import jakarta.xml.bind.JAXBElement; /** * @author António Lindo @@ -23,6 +25,11 @@ public SIARDDK1007DocIndexFileStrategy() { super(); } + @Override + JAXBElement createDocIndexTypeRootInstance() { + return new ObjectFactory().createDocIndex(createDocIndexTypeInstance()); + } + @Override DocIndexType createDocIndexTypeInstance() { return new DocIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java index a80f8276f..c6e66d8b0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java @@ -18,6 +18,8 @@ import java.util.List; +import dk.sa.xmlns.diark._1_0.fileindex.ObjectFactory; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +38,11 @@ public SIARDDK1007FileIndexFileStrategy() { super(); } + @Override + JAXBElement createFileIndexTypeRootInstance() { + return new ObjectFactory().createFileIndex(createFileIndexTypeInstance()); + } + @Override FileIndexType createFileIndexTypeInstance() { return new FileIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java index c57ae786c..259518690 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java @@ -7,11 +7,14 @@ */ package com.databasepreservation.modules.siard.out.metadata; +import java.math.BigInteger; +import java.util.List; + import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocumentType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.ObjectFactory; -import java.math.BigInteger; -import java.util.List; +import jakarta.xml.bind.JAXBElement; /** * @author António Lindo @@ -23,6 +26,11 @@ public SIARDDK128DocIndexFileStrategy() { super(); } + @Override + JAXBElement createDocIndexTypeRootInstance() { + return new ObjectFactory().createDocIndex(createDocIndexTypeInstance()); + } + @Override DocIndexType createDocIndexTypeInstance() { return new DocIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java index d17924986..723712579 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java @@ -17,6 +17,8 @@ package com.databasepreservation.modules.siard.out.metadata; import com.databasepreservation.modules.siard.bindings.siard_dk_128.FileIndexType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.ObjectFactory; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,11 @@ public SIARDDK128FileIndexFileStrategy() { super(); } + @Override + JAXBElement createFileIndexTypeRootInstance() { + return new ObjectFactory().createFileIndex(createFileIndexTypeInstance()); + } + @Override FileIndexType createFileIndexTypeInstance() { return new FileIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java new file mode 100644 index 000000000..5b28bd2d3 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java @@ -0,0 +1,125 @@ +package com.databasepreservation.modules.siard.out.metadata; + +import java.io.IOException; +import java.io.OutputStream; + +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.model.structure.DatabaseStructure; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; +import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; +import com.databasepreservation.modules.siard.common.adapters.SIARDDKAdapter; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; +import com.databasepreservation.modules.siard.out.write.WriteStrategy; + +/** + * + * @author Alexandre Flores + */ +public class SIARDDK128MetadataExportStrategy extends SIARDDKMetadataExportStrategy { + + public SIARDDK128MetadataExportStrategy(SIARDDKExportModule siarddkExportModule, SIARDDKAdapter siarddkAdapter) { + super(siarddkExportModule, siarddkAdapter); + } + + @Override + public void writeMetadataXML(DatabaseStructure dbStructure, SIARDArchiveContainer outputContainer, + WriteStrategy writeStrategy) throws ModuleException { + // TO-DO: Refactor this into one method in class that can be used by + // SIARDDKDatabaseExportModule also + + // Generate tableIndex.xml + + try { + IndexFileStrategy tableIndexFileStrategy = new SIARDDKTableIndexFileStrategy(lobsTracker, siarddkAdapter); + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + + siardMarshaller.marshal("com.databasepreservation.modules.siard.bindings.siard_dk_128", + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.TABLE_INDEX), + "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/tableIndex.xsd", writer, + tableIndexFileStrategy.generateXML(dbStructure)); + + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing tableIndex.xml to the archive.").withCause(e); + } + + // Generate archiveIndex.xml + + if (exportModuleArgs.get(SIARDDKConstants.ARCHIVE_INDEX) != null) { + try { + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.ARCHIVE_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + IndexFileStrategy archiveIndexFileStrategy = new CommandLineIndexFileStrategy(SIARDDKConstants.ARCHIVE_INDEX, + exportModuleArgs, writer, metadataPathStrategy); + archiveIndexFileStrategy.generateXML(null); + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing archiveIndex.xml to the archive").withCause(e); + } + } + + // Generate contextDocumentationIndex.xml + + if (exportModuleArgs.get(SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX) != null) { + try { + + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + IndexFileStrategy contextDocumentationIndexFileStrategy = new CommandLineIndexFileStrategy( + SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, exportModuleArgs, writer, metadataPathStrategy); + contextDocumentationIndexFileStrategy.generateXML(null); + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing contextDocumentationIndex.xml to the archive") + .withCause(e); + } + } + + if (lobsTracker.getLOBsCount() > 0) { + try { + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + + siardMarshaller.marshal(DocIndexType.class, metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.DOC_INDEX), + "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/docIndex.xsd", writer, + SIARDDKDocIndexFileStrategy.generateXML(dbStructure)); + + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing docIndex.xml to the archive.").withCause(e); + } + } + + createLocalSharedFolder(outputContainer); + } + + @Override + public void writeMetadataXSD(DatabaseStructure dbStructure, SIARDArchiveContainer outputContainer, + WriteStrategy writeStrategy) throws ModuleException { + + // Write contents to Schemas/standard + writeSchemaFile(outputContainer, SIARDDKConstants.XML_SCHEMA, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.TABLE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.ARCHIVE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.FILE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.RESEARCH_INDEX, writeStrategy); + if (lobsTracker.getLOBsCount() > 0) { + writeSchemaFile(outputContainer, SIARDDKConstants.DOC_INDEX, writeStrategy); + } + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java index 62e393839..c0df07692 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java @@ -12,6 +12,7 @@ import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.structure.DatabaseStructure; +import jakarta.xml.bind.JAXBElement; /** * @author Andreas Kring @@ -19,10 +20,10 @@ */ public abstract class SIARDDKDocIndexFileStrategy implements IndexFileStrategy { - private T docIndex; + private JAXBElement docIndex; public SIARDDKDocIndexFileStrategy() { - docIndex = createDocIndexTypeInstance(); + docIndex = createDocIndexTypeRootInstance(); } /* @@ -70,11 +71,13 @@ public D addDoc(int dID, int pID, int mID, int docCollectionNumber, String oFn, setGmlXsd(doc, gmlXsd); } - getDoc(docIndex).add(doc); + getDoc(docIndex.getValue()).add(doc); return doc; } + abstract JAXBElement createDocIndexTypeRootInstance(); + abstract T createDocIndexTypeInstance(); abstract D createDocumentTypeInstance(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java index 17b8a3ed8..beaf91773 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java @@ -12,6 +12,7 @@ import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,8 +57,8 @@ public Object generateXML(DatabaseStructure dbStructure) throws ModuleException String foNbase = baseContainer.getName(count - 1).toString(); // e.g. // AVID.SA.19000.1 - T fileIndexType = createFileIndexTypeInstance(); - List fList = getF(fileIndexType); + JAXBElement fileIndexType = createFileIndexTypeRootInstance(); + List fList = getF(fileIndexType.getValue()); for (Map.Entry entry : md5sums.entrySet()) { @@ -168,6 +169,8 @@ public byte[] addFile(String path) { return digest; } + abstract JAXBElement createFileIndexTypeRootInstance(); + abstract T createFileIndexTypeInstance(); abstract D createFileIndexTypeFInstance(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java index 6856cc368..53de790e0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java @@ -7,6 +7,18 @@ */ package com.databasepreservation.modules.siard.out.metadata; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.reporters.Reporter; import com.databasepreservation.model.structure.DatabaseStructure; @@ -17,17 +29,6 @@ import com.databasepreservation.modules.siard.out.content.LOBsTracker; import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.write.WriteStrategy; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; /** * @author Andreas Kring @@ -36,13 +37,13 @@ public class SIARDDKMetadataExportStrategy implements MetadataExportStrategy { private static final Logger LOGGER = LoggerFactory.getLogger(SIARDDKMetadataExportStrategy.class); - private SIARDMarshaller siardMarshaller; - private MetadataPathStrategy metadataPathStrategy; - private SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; - private SIARDDKDocIndexFileStrategy SIARDDKDocIndexFileStrategy; - private Map exportModuleArgs; - private LOBsTracker lobsTracker; - private SIARDDKAdapter siarddkAdapter; + protected SIARDMarshaller siardMarshaller; + protected MetadataPathStrategy metadataPathStrategy; + protected SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; + protected SIARDDKDocIndexFileStrategy SIARDDKDocIndexFileStrategy; + protected Map exportModuleArgs; + protected LOBsTracker lobsTracker; + protected SIARDDKAdapter siarddkAdapter; private Reporter reporter; @@ -164,7 +165,7 @@ public void setOnceReporter(Reporter reporter) { this.reporter = reporter; } - private void writeSchemaFile(SIARDArchiveContainer container, String indexFile, WriteStrategy writeStrategy) + protected void writeSchemaFile(SIARDArchiveContainer container, String indexFile, WriteStrategy writeStrategy) throws ModuleException { InputStream inputStream = this.getClass().getResourceAsStream(metadataPathStrategy.getXsdResourcePath(indexFile)); @@ -197,7 +198,7 @@ private void writeSchemaFile(SIARDArchiveContainer container, String indexFile, } } - private void createLocalSharedFolder(SIARDArchiveContainer container) { + protected void createLocalSharedFolder(SIARDArchiveContainer container) { Path containerPath = container.getPath(); Path localShared = Paths.get("Schemas/localShared"); File folder = containerPath.resolve(localShared).toFile(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java index ec54b1d78..d6efe5353 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java @@ -33,4 +33,21 @@ public interface SIARDMarshaller { */ public void marshal(String context, String localeSchemaLocation, String JAXBSchemaLocation, OutputStream writer, Object jaxbElement) throws ModuleException; + + /** + * Generate JAXB Marshaller for writing XML object to the archive. + * + * @param archiveClass + * Siard archive class to give JAXB context + * @param localeSchemaLocation + * The locale location of the XML schema for the metadata file. + * @param JAXBSchemaLocation + * The Marshaller.JAXB_SCHEMA_LOCATION. + * @param writer + * The OutputStream to write to. + * @param jaxbElement + * The JAXB element to marshal. + */ + public void marshal(Class archiveClass, String localeSchemaLocation, String JAXBSchemaLocation, + OutputStream writer, Object jaxbElement) throws ModuleException; } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java index fa1d86f47..1a8dfc108 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java @@ -12,9 +12,6 @@ import java.io.OutputStream; import javax.xml.XMLConstants; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Marshaller; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; @@ -26,6 +23,10 @@ import com.databasepreservation.model.exception.ModuleException; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; + public class StandardSIARDMarshaller implements SIARDMarshaller { private static final String ENCODING = "UTF-8"; @@ -78,4 +79,51 @@ public void marshal(String contextStr, String localeSchemaLocation, String JAXBS throw new ModuleException().withMessage("Error while Marshalling JAXB").withCause(e); } } + + @Override + public void marshal(Class archiveClass, String localeSchemaLocation, String JAXBSchemaLocation, + OutputStream writer, Object jaxbElement) throws ModuleException { + + // Set up JAXB marshaller + + JAXBContext context; + try { + context = JAXBContext.newInstance(archiveClass.getPackage().getName(), archiveClass.getClassLoader()); + } catch (JAXBException e) { + throw new ModuleException().withMessage("Error loading JAXBContent").withCause(e); + } + + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema xsdSchema = null; + try { + InputStream in = this.getClass().getResourceAsStream(localeSchemaLocation); + xsdSchema = schemaFactory.newSchema(new StreamSource(in)); + in.close(); + } catch (SAXException e) { + throw new ModuleException() + .withMessage("XSD file has errors: " + getClass().getResource(localeSchemaLocation).getPath()).withCause(e); + } catch (IOException e) { + throw new ModuleException().withMessage("Could not close InputStream").withCause(e); + } + + Marshaller m; + + try { + + m = context.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + m.setProperty(Marshaller.JAXB_ENCODING, ENCODING); + m.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, JAXBSchemaLocation); + + m.setSchema(xsdSchema); + + m.marshal(jaxbElement, writer); + + } catch (JAXBException e) { + if (e.getCause() instanceof SAXParseException) { + LOGGER.error(e.getCause().getMessage()); + } + throw new ModuleException().withMessage("Error while Marshalling JAXB").withCause(e); + } + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java index d251c3570..cac3d926d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java @@ -9,6 +9,8 @@ import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import dk.sa.xmlns.diark._1_0.fileindex.FileIndexType; + /** * @author António Lindo * @@ -23,4 +25,9 @@ public SIARDDK1007DatabaseExportModule(SIARDDKExportModule siarddkExportModule) String getJAXBContext() { return SIARDDKConstants.JAXB_CONTEXT_FILEINDEX; } + + @Override + Class getJAXBContextClass() { + return FileIndexType.class; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java index 334aa2017..ab8b88353 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java @@ -7,6 +7,7 @@ */ package com.databasepreservation.modules.siard.out.output; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.SiardDiark; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; /** @@ -23,4 +24,9 @@ public SIARDDK128DatabaseExportModule(SIARDDKExportModule siarddkExportModule) { String getJAXBContext() { return SIARDDKConstants.JAXB_CONTEXT_FILEINDEX_128; } + + @Override + Class getJAXBContextClass() { + return SiardDiark.class; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java index 0c70f2856..9d1876645 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java @@ -14,15 +14,17 @@ package com.databasepreservation.modules.siard.out.output; +import java.util.Map; + import com.databasepreservation.modules.siard.common.adapters.SIARDDK128Adapter; import com.databasepreservation.modules.siard.common.path.SIARDDK128MetadataPathStrategy; import com.databasepreservation.modules.siard.common.path.SIARDDKMetadataPathStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDK128DocIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDK128FileIndexFileStrategy; +import com.databasepreservation.modules.siard.out.metadata.SIARDDK128MetadataExportStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKDocIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKMetadataExportStrategy; -import java.util.Map; /** * @author António Lindo @@ -51,7 +53,7 @@ SIARDDKMetadataPathStrategy createSIARDDKMetadataPathStrategyInstance() { @Override SIARDDKMetadataExportStrategy createSIARDDKMetadataExportStrategyInstance() { - return new SIARDDKMetadataExportStrategy(this, new SIARDDK128Adapter()); + return new SIARDDK128MetadataExportStrategy(this, new SIARDDK128Adapter()); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index e9a85f2cc..d750ca8c2 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -7,33 +7,78 @@ */ package com.databasepreservation.modules.siard.out.output; -import com.databasepreservation.model.exception.ModuleException; -import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; -import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; -import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; -import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.common.io.providers.PathInputStreamProvider; +import com.databasepreservation.model.data.BinaryCell; +import com.databasepreservation.model.data.Cell; +import com.databasepreservation.model.data.Row; +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.SIARDDKModuleFactory; +import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; +import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; +import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; +import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionService; +import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionServiceException; +import com.databasepreservation.modules.siard.services.conversion.LobConversionService; +import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; +import com.databasepreservation.utils.ConfigUtils; /** + * Handles database export pipeline matching SIARD-DK compliance, incorporating + * high-throughput asynchronous LOB streaming conversion. + * * @author Andreas Kring * */ public abstract class SIARDDKDatabaseExportModule extends SIARDExportDefault { - private SIARDDKExportModule siarddkExportModule; + private final SIARDDKExportModule siarddkExportModule; private static final Logger logger = LoggerFactory.getLogger(SIARDDKDatabaseExportModule.class); + private ExecutorService executorService; + private TempFileTracker tempFileTracker; + private LobConversionService conversionService; + private String targetLobFormat; + private static final Integer MAX_QUEUE_SIZE = ConfigUtils.getProperty(100, "dbptk.siarddk.export.maxQueueSize"); + + // Resilient Pipeline architecture attributes + private BlockingQueue> pendingRowsQueue; + private ExecutorService writerExecutor; + private Future writerTask; + private final AtomicReference writerError = new AtomicReference<>(); + + /** + * Data context tuple to link rows with their transient extracted disk paths. + */ + private record ProcessedRowContext(Row row, List transientPaths) { + } + public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { super(siarddkExportModule.getContentExportStrategy(), siarddkExportModule.getMainContainer(), siarddkExportModule.getWriteStrategy(), siarddkExportModule.getMetadataExportStrategy(), null); @@ -45,6 +90,29 @@ public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { public void initDatabase() throws ModuleException { super.initDatabase(); + Map exportModuleArgs = siarddkExportModule.getExportModuleArgs(); + boolean isLobConversionEnabled = Boolean + .parseBoolean(exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_ENABLED, "false")); + + if (isLobConversionEnabled) { + this.tempFileTracker = new TempFileTracker(); + String apiEndpoint = exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_ENDPOINT, + "http://localhost:8087"); + this.targetLobFormat = exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_TARGET_FORMAT, + "image/tiff"); + + this.conversionService = new HttpLobConversionService(apiEndpoint, this.targetLobFormat, this.tempFileTracker); + logger.info("LOB conversion service enabled. Endpoint: '{}', Target Format: '{}'", apiEndpoint, + this.targetLobFormat); + } else { + this.conversionService = null; + this.tempFileTracker = null; + logger.info("LOB conversion service is disabled."); + } + + this.executorService = Executors.newVirtualThreadPerTaskExecutor(); + this.writerExecutor = Executors.newSingleThreadExecutor(); // Dedicated single-thread pipeline consumer + // Get docID info from the command line and add these to the LOBsTracker Path pathToArchive = siarddkExportModule.getMainContainer().getPath(); @@ -82,8 +150,39 @@ public void initDatabase() throws ModuleException { } } + @Override + public void handleDataOpenTable(String tableId) throws ModuleException { + logger.debug("Opening table '{}'. Initializing asynchronous pipeline...", tableId); + this.pendingRowsQueue = new LinkedBlockingQueue<>(MAX_QUEUE_SIZE); + this.writerError.set(null); + startAsyncWriter(); + super.handleDataOpenTable(tableId); + } + + @Override + public void handleDataRow(Row row) throws ModuleException { + enqueueRow(row); + } + + @Override + public void handleDataCloseTable(String tableId) throws ModuleException { + logger.debug("Closing table '{}'. Draining remaining items in the pipeline...", tableId); + stopAsyncWriter(); + super.handleDataCloseTable(tableId); + } + @Override public void finishDatabase() throws ModuleException { + if (executorService != null && !executorService.isShutdown()) { + executorService.shutdown(); + } + if (writerExecutor != null && !writerExecutor.isShutdown()) { + writerExecutor.shutdown(); + } + if (tempFileTracker != null) { + tempFileTracker.cleanupAll(); + } + super.finishDatabase(); // Write ContextDocumentation to archive @@ -117,7 +216,8 @@ public void finishDatabase() throws ModuleException { OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(siarddkExportModule.getMainContainer(), path, siarddkExportModule.getWriteStrategy()); - siardMarshaller.marshal(getJAXBContext(), metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), + siardMarshaller.marshal(getJAXBContextClass(), + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/fileIndex.xsd", writer, SIARDDKFileIndexFileStrategy.generateXML(null)); @@ -125,8 +225,187 @@ public void finishDatabase() throws ModuleException { } catch (IOException e) { throw new ModuleException().withMessage("Error writing fileIndex to the archive.").withCause(e); } + } + /** + * Starts the sequential pipeline background consumer thread. + */ + private void startAsyncWriter() { + this.writerTask = writerExecutor.submit(() -> { + try { + while (!Thread.currentThread().isInterrupted()) { + logger.debug("Consumer thread is waiting to take the oldest row from the queue..."); + Future future = pendingRowsQueue.take(); // Enforces strict sequential order + + logger.debug("Oldest row taken. Awaiting its Virtual Thread completion (LOB HTTP boundary)..."); + ProcessedRowContext context = future.get(); // Awaits specific LOB HTTP processing boundary + + if (context == null) { + logger.debug("<< DEQUEUED: Poison Pill received. Safely shutting down the consumer thread."); + break; + } + + super.handleDataRow(context.row()); + + // Alleviate disk pressure by wiping extracted structures instantly after XML + // writing + cleanupTransientPaths(context.transientPaths()); + logger.debug("<< DEQUEUED: Row [{}] removed from queue and successfully written. Current size: {}/{}", + context.row().getIndex(), pendingRowsQueue.size(), MAX_QUEUE_SIZE); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Pipeline consumer thread was interrupted and is shutting down."); + } catch (ExecutionException e) { + logger.error("A background conversion task failed critically: {}", e.getCause().getMessage()); + writerError.set(e.getCause()); + } catch (Exception e) { + logger.error("An unexpected error occurred during sequential writing.", e); + writerError.set(e); + } + }); + } + + /** + * Gracefully drains the remaining queue, safely stops the consumer and monitors + * failures. + */ + private void stopAsyncWriter() throws ModuleException { + if (writerError.get() == null) { + try { + logger.debug("|| TABLE END: Injecting Poison Pill into the queue and waiting for consumer to finish..."); + + Future poisonPill = executorService.submit(() -> null); + + while (!pendingRowsQueue.offer(poisonPill, 500, TimeUnit.MILLISECONDS)) { + if (writerError.get() != null) + break; + } + + if (writerTask != null) { + writerTask.get(); + } + + } catch (InterruptedException | ExecutionException e) { + Thread.currentThread().interrupt(); + throw new ModuleException().withMessage("Failed to cleanly stop the background writer").withCause(e); + } + } + + // Purge any remaining futures in case of a catastrophic error + if (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { + for (Future future : pendingRowsQueue) { + future.cancel(true); + } + pendingRowsQueue.clear(); + } + + if (writerError.get() != null) { + throw new ModuleException().withMessage("Row writing pipeline aborted").withCause(writerError.get()); + } + } + + /** + * Pushes rows into the pipeline, throwing fast if the writer task fails, and + * using non-deadlocking backpressure. + */ + private void enqueueRow(Row row) throws ModuleException { + if (writerError.get() != null) { + throw new ModuleException().withMessage("Pipeline execution halted due to previous background failure") + .withCause(writerError.get()); + } + + Callable conversionTask = () -> processRowAsync(row); + Future future = executorService.submit(conversionTask); + + try { + // Prevents deadlocks if the consumer thread crashes while queue is maxed out + boolean waitingLogged = false; + while (!pendingRowsQueue.offer(future, 500, TimeUnit.MILLISECONDS)) { + if (writerError.get() != null) { + future.cancel(true); + throw new ModuleException().withMessage("Pipeline halted while enqueuing row").withCause(writerError.get()); + } + + if (!waitingLogged) { + logger.debug("|| PAUSED: Queue is full. Suspended database reading. Waiting for space to enqueue row [{}]...", + row.getIndex()); + waitingLogged = true; + } + } + logger.debug(">> ENQUEUED: Row [{}] entered the queue. Current size: {}/{}", row.getIndex(), + pendingRowsQueue.size(), MAX_QUEUE_SIZE); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + future.cancel(true); + throw new ModuleException().withMessage("Row enqueuing process was interrupted").withCause(e); + } + } + + /** + * Processes row columns concurrently on Virtual Threads mapping extracted files + * for lifecycle control. + */ + private ProcessedRowContext processRowAsync(Row row) throws ModuleException { + List cells = row.getCells(); + List transientPaths = new ArrayList<>(); + + if (this.conversionService != null) { + for (int i = 0; i < cells.size(); i++) { + Cell cell = cells.get(i); + + if (cell instanceof BinaryCell binCell) { + try (InputStream originalStream = binCell.createInputStream()) { + ConversionResult result = conversionService.convertLob(cell.getId(), originalStream); + + // TODO: Handle multiple files per cell if needed. Currently assumes single file + // output. + BinaryCell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.zipFile()), + "application/zip"); + newCell.setFile(binCell.getFile()); + cells.set(i, newCell); + + // Track extracted parts to clean them individually later + transientPaths.addAll(result.convertedFiles()); + transientPaths.add(result.reportFile()); + transientPaths.add(result.zipFile()); + if (!result.convertedFiles().isEmpty()) { + transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container + } + } catch (IOException | InterruptedException | HttpLobConversionServiceException e) { + String statusCodeInfo = ""; + if (e instanceof HttpLobConversionServiceException apiEx && apiEx.getHttpStatusCode() != null) { + statusCodeInfo = " (HTTP " + apiEx.getHttpStatusCode() + ")"; + } + + String errorMsg = String.format( + "Conversion failed for cell '%s' in row %d%s. " + "Pipeline continuing with original file. Detail: %s", + cell.getId(), row.getIndex(), statusCodeInfo, e.getMessage()); + + logger.error(errorMsg); + } + } + } + } + row.setCells(cells); + + long readyCount = pendingRowsQueue.stream().filter(Future::isDone).count(); + logger.debug("== READY: Row [{}] finished conversion. Currently {} ready rows waiting in queue.", row.getIndex(), + readyCount + 1); + + return new ProcessedRowContext(row, transientPaths); + } + + private void cleanupTransientPaths(List paths) { + if (paths == null || tempFileTracker == null) { + return; + } + for (Path path : paths) { + tempFileTracker.deleteEarly(path); + } } abstract String getJAXBContext(); -} + + abstract Class getJAXBContextClass(); +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java index 7fd8d54d0..85c7e2dc6 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java @@ -35,14 +35,14 @@ * */ public abstract class SIARDDKExportModule { - private MetadataExportStrategy metadataExportStrategy; - private SIARDArchiveContainer mainContainer; - private ContentExportStrategy contentExportStrategy; - private WriteStrategy writeStrategy; - private ContentPathExportStrategy contentPathExportStrategy; - private MetadataPathStrategy metadataPathStrategy; - private SIARDMarshaller siardMarshaller; - private LOBsTracker lobsTracker; + protected MetadataExportStrategy metadataExportStrategy; + protected SIARDArchiveContainer mainContainer; + protected ContentExportStrategy contentExportStrategy; + protected WriteStrategy writeStrategy; + protected ContentPathExportStrategy contentPathExportStrategy; + protected MetadataPathStrategy metadataPathStrategy; + protected SIARDMarshaller siardMarshaller; + protected LOBsTracker lobsTracker; private Map exportModuleArgs; private SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java index dcbb3fce0..98731d8f1 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java @@ -7,10 +7,11 @@ */ package com.databasepreservation.modules.siard.out.path; +import org.apache.commons.lang3.NotImplementedException; + import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.content.LOBsTracker; import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; -import org.apache.commons.lang3.NotImplementedException; /** * @author Andreas Kring @@ -24,14 +25,13 @@ public class SIARDDKContentPathExportStrategy implements ContentPathExportStrate private static final String SCHEMA_DIR = "schema"; private static final String DOCUMENT_DIR = "Documents"; private static final String DOC_COLLECTION = "docCollection"; - private static final String fileCount = "1"; // Design decision private LOBsTracker lobsTracker; public SIARDDKContentPathExportStrategy(LOBsTracker loBsTracker) { this.lobsTracker = loBsTracker; } - + public SIARDDKContentPathExportStrategy(SIARDDKExportModule siarddkExportModule) { this.lobsTracker = siarddkExportModule.getLobsTracker(); } @@ -71,13 +71,12 @@ public String getBlobFilePath(int schemaIndex, int tableIndex, int columnIndex, // TO-DO: add test case int docCollectionCount = lobsTracker.getDocCollectionCount(); - int LOBsCount = lobsTracker.getLOBsCount(); + int LOBsCount = lobsTracker.getLOBsCount() + 1; // Note: code assumes one file in each folder return new StringBuilder().append(DOCUMENT_DIR).append(SIARDDKConstants.FILE_SEPARATOR).append(DOC_COLLECTION) .append(docCollectionCount).append(SIARDDKConstants.FILE_SEPARATOR).append(LOBsCount) - .append(SIARDDKConstants.FILE_SEPARATOR).append(fileCount).append(SIARDDKConstants.FILE_EXTENSION_SEPARATOR) - .toString(); + .append(SIARDDKConstants.FILE_SEPARATOR).toString(); } // Not used in SIARDDK diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java new file mode 100644 index 000000000..bf20cadd6 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -0,0 +1,252 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatus; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatusResponse; +import com.databasepreservation.modules.siard.services.conversion.model.JobSubmissionResponse; +import com.databasepreservation.utils.ConfigUtils; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Handles communication with the external format conversion REST API. Uses + * asynchronous polling and in-memory multipart streaming to process large + * objects efficiently. + * + * @author Gabriel Barros + */ +public class HttpLobConversionService implements LobConversionService { + + private static final Logger log = LoggerFactory.getLogger(HttpLobConversionService.class); + + private static final int MAX_NETWORK_RETRIES = ConfigUtils.getProperty(3, + "dbptk.service.lob.conversion.networkRetries"); + private static final int BASE_POLLING_INTERVAL_MS = ConfigUtils.getProperty(2000, + "dbptk.service.lob.conversion.basePollingIntervalMs"); + private static final int MAX_POLLING_ATTEMPTS = ConfigUtils.getProperty(600, + "dbptk.service.lob.conversion.maxPollingAttempts"); + private static final int CONNECTION_TIMEOUT_SECONDS = ConfigUtils.getProperty(60, + "dbptk.service.lob.conversion.connectionTimeoutSeconds"); + + private static final String CRLF = "\r\n"; + + private final HttpClient httpClient; + private final String baseUrl; + private final String targetFormat; + private final ObjectMapper objectMapper; + private final TempFileTracker fileTracker; + private final Random random = new Random(); + + public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTracker fileTracker) { + this.baseUrl = baseUrl; + this.targetFormat = targetFormat; + this.fileTracker = fileTracker; + this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(CONNECTION_TIMEOUT_SECONDS)).build(); + } + + @Override + public ConversionResult convertLob(String cellId, InputStream inputStream) + throws IOException, InterruptedException, HttpLobConversionServiceException { + log.debug("Initiating conversion pipeline for cell: {}", cellId); + String jobId = submitJob(cellId, inputStream); + waitForCompletion(cellId, jobId); + return downloadResult(cellId, jobId); + } + + private String submitJob(String cellId, InputStream inputStream) + throws IOException, InterruptedException, HttpLobConversionServiceException { + String boundary = "DbptkBoundary" + System.currentTimeMillis(); + + String header = buildMultipartHeader(boundary, cellId); + String footer = CRLF + "--" + boundary + "--" + CRLF; + + InputStream headerStream = new ByteArrayInputStream(header.getBytes(StandardCharsets.UTF_8)); + InputStream footerStream = new ByteArrayInputStream(footer.getBytes(StandardCharsets.UTF_8)); + + InputStream multipartStream = new SequenceInputStream(new SequenceInputStream(headerStream, inputStream), + footerStream); + + HttpRequest submitRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs")) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(BodyPublishers.ofInputStream(() -> multipartStream)).build(); + + HttpResponse submitResponse = executeWithRetry(submitRequest, BodyHandlers.ofString(), MAX_NETWORK_RETRIES); + + if (submitResponse.statusCode() >= 400) { + log.error("API rejected LOB submission for cell {}. Status: {}, Body: {}", cellId, submitResponse.statusCode(), + submitResponse.body()); + throw new HttpLobConversionServiceException("Failed to submit LOB for cell " + cellId, + submitResponse.statusCode()); + } + + JobSubmissionResponse job = objectMapper.readValue(submitResponse.body(), JobSubmissionResponse.class); + log.debug("Successfully dispatched cell {}. Assigned Job ID: {}", cellId, job.id()); + return job.id(); + } + + private void waitForCompletion(String cellId, String jobId) + throws IOException, InterruptedException, HttpLobConversionServiceException { + log.debug("Awaiting completion of Job {} (Cell {})", jobId, cellId); + + for (int attempts = 1; attempts <= MAX_POLLING_ATTEMPTS; attempts++) { + HttpRequest statusRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).GET().build(); + HttpResponse statusResponse = executeWithRetry(statusRequest, BodyHandlers.ofString(), + MAX_NETWORK_RETRIES); + + JobStatusResponse response = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); + + switch (response.status()) { + case JobStatus.COMPLETED -> { + log.debug("Job {} (Cell {}) completed successfully after {} attempts.", jobId, cellId, attempts); + return; + } + case JobStatus.FAILED, JobStatus.EVICTED -> { + log.error("API reported terminal failure for Job {} (Cell {}) with status: {}", jobId, cellId, + response.status()); + throw new HttpLobConversionServiceException( + "Server failed to convert cell: " + cellId + " (Status: " + response.status() + ")"); + } + case JobStatus.ACCEPTED, JobStatus.PROCESSING -> { + if (attempts % 30 == 0) { + log.warn("Job {} (Cell {}) is taking unusually long. Current status: {}. Attempt: {}/{}", jobId, cellId, + response.status(), attempts, MAX_POLLING_ATTEMPTS); + } + long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); + Thread.sleep(sleepTime); + } + } + } + log.error("Zombie Job detected. API failed to resolve Job {} (Cell {}) within the maximum polling threshold.", + jobId, cellId); + throw new HttpLobConversionServiceException("Timeout after waiting for conversion of cell: " + cellId); + } + + /** + * Downloads the resulting ZIP and lists its contents, returning the compressed + * file. + */ + private ConversionResult downloadResult(String cellId, String jobId) throws IOException, InterruptedException { + Path zipFile = Files.createTempFile("siarddk_conv_" + cellId + "_" + jobId, ".zip"); + + fileTracker.track(zipFile); + + HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) + .GET().build(); + + executeWithRetry(downloadRequest, BodyHandlers.ofFile(zipFile), MAX_NETWORK_RETRIES); + + return listZipContents(cellId, zipFile); + } + + private ConversionResult listZipContents(String cellId, Path zipFile) throws IOException { + Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); + fileTracker.trackDir(extractionDir); + + Path normalizedExtractionDir = extractionDir.normalize(); + + List convertedFiles = new ArrayList<>(); + Path reportFile = null; + + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile.toFile()))) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + Path extractedFilePath = extractionDir.resolve(zipEntry.getName()).normalize(); + + if (!extractedFilePath.startsWith(normalizedExtractionDir)) { + throw new SecurityException("Corrupted ZIP entry (Zip Slip vulnerability detected): " + zipEntry.getName()); + } + + if (!zipEntry.isDirectory()) { + Files.copy(zis, extractedFilePath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + + if (zipEntry.getName().toLowerCase().contains("report")) { + reportFile = extractedFilePath; + } else { + convertedFiles.add(extractedFilePath); + } + } + } + } + + if (convertedFiles.isEmpty() || reportFile == null) { + throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); + } + + return new ConversionResult(convertedFiles, reportFile, zipFile); + } + + private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, + int maxRetries) throws InterruptedException, IOException { + Exception lastException = null; + + for (int attempt = 1; attempt <= maxRetries; attempt++) { + try { + HttpResponse response = httpClient.send(request, responseBodyHandler); + + if (response.statusCode() >= 500) { + throw new IOException("Temporary server error: " + response.statusCode() + " - " + response.body()); + } + + return response; + } catch (IOException e) { + lastException = e; + log.warn("Attempt {} failed for {}: {}", attempt, request.uri(), e.getMessage()); + + if (attempt == maxRetries) + break; + + try { + Thread.sleep((long) Math.pow(2, attempt) * 1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Retry interrupted for URI: " + request.uri(), ie); + } + } + } + throw new IOException("Exhausted all network retries for URI: " + request.uri(), lastException); + } + + private String buildMultipartHeader(String boundary, String cellId) { + StringBuilder sb = new StringBuilder(); + + // Target format part header + sb.append("--").append(boundary).append(CRLF); + sb.append("Content-Disposition: form-data; name=\"targetFormat\"").append(CRLF); + sb.append(CRLF); + sb.append(this.targetFormat).append(CRLF); + + // File part header + sb.append("--").append(boundary).append(CRLF); + sb.append("Content-Disposition: form-data; name=\"file\"; filename=\"lob_").append(cellId).append(".bin\"") + .append(CRLF); + sb.append("Content-Type: application/octet-stream").append(CRLF); + sb.append(CRLF); + + return sb.toString(); + } +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java new file mode 100644 index 000000000..9fc7d9960 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java @@ -0,0 +1,33 @@ +package com.databasepreservation.modules.siard.services.conversion; + +/** + * @author Gabriel Barros + */ +public class HttpLobConversionServiceException extends Exception { + + private final Integer httpStatusCode; + + public HttpLobConversionServiceException(String message) { + super(message); + this.httpStatusCode = null; + } + + public HttpLobConversionServiceException(String message, Throwable cause) { + super(message, cause); + this.httpStatusCode = null; + } + + public HttpLobConversionServiceException(String message, int httpStatusCode) { + super(message); + this.httpStatusCode = httpStatusCode; + } + + public HttpLobConversionServiceException(String message, int httpStatusCode, Throwable cause) { + super(message, cause); + this.httpStatusCode = httpStatusCode; + } + + public Integer getHttpStatusCode() { + return httpStatusCode; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java new file mode 100644 index 000000000..a5d4d30d1 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java @@ -0,0 +1,41 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Enriches and persists LOB conversion reports. + * + * @author Gabriel Barros + */ +public class LobConversionAuditor { + private static final Logger logger = LoggerFactory.getLogger(LobConversionAuditor.class); + private final ObjectMapper mapper; + private final Path auditFilePath; + + public LobConversionAuditor(Path baseExportDirectory, String archiveName) { + this.mapper = new ObjectMapper(); + String fileName = archiveName + "_lob_conversion_audit.jsonl"; + this.auditFilePath = baseExportDirectory.resolve(fileName); + } + + public void appendAuditRecord(ConversionReport report) { + if (report == null) + return; + + try { + String jsonLine = mapper.writeValueAsString(report); + Files.writeString(auditFilePath, jsonLine + System.lineSeparator(), StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (Exception e) { + logger.error("Failed to append conversion report to audit log.", e); + } + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java new file mode 100644 index 000000000..0f26c5b32 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java @@ -0,0 +1,12 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.IOException; +import java.io.InputStream; + +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; + +public interface LobConversionService { + ConversionResult convertLob(String cellId, InputStream inputStream) + throws IOException, ModuleException, InterruptedException, HttpLobConversionServiceException; +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java new file mode 100644 index 000000000..ae82d21e0 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java @@ -0,0 +1,95 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tracks and manages the lifecycle of temporary files and directories created + * during the LOB conversion process. + * + * @author Gabriel Barros + */ +public class TempFileTracker { + private static final Logger LOGGER = LoggerFactory.getLogger(TempFileTracker.class); + private final Queue trackedPaths = new ConcurrentLinkedQueue<>(); + + /** + * Registers a file or directory path to be tracked for final cleanup. * @param + * path The path to track. + */ + public void track(Path path) { + if (path != null) { + trackedPaths.add(path); + } + } + + /** + * Registers a directory path to be tracked for final cleanup. * @param dirPath + * The directory path to track. + */ + public void trackDir(Path dirPath) { + track(dirPath); + } + + /** + * Deletes a tracked path immediately (including non-empty directories) and + * removes it from the tracking queue to free resources early. * @param path The + * path to delete immediately. + */ + public void deleteEarly(Path path) { + if (path == null) { + return; + } + try { + deleteRecursively(path); + trackedPaths.remove(path); + } catch (Exception e) { + LOGGER.warn("Unable to delete temporary path early: " + path, e); + } + } + + /** + * Deletes all remaining tracked files and directories comprehensively, ensuring + * recursive cleanup of nested content. + */ + public void cleanupAll() { + for (Path path : trackedPaths) { + try { + deleteRecursively(path); + } catch (Exception e) { + LOGGER.warn("Unable to delete temporary file/directory during bulk cleanup: " + path, e); + } + } + trackedPaths.clear(); + } + + /** + * Helper method to perform safe recursive deletion of paths and directories. + */ + private void deleteRecursively(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + if (Files.isDirectory(path)) { + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException e) { + LOGGER.warn("Failed to delete nested path: " + p, e); + } + }); + } + } else { + Files.deleteIfExists(path); + } + } +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java new file mode 100644 index 000000000..5d229bb2d --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java @@ -0,0 +1,10 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +import java.nio.file.Path; +import java.util.List; + +/** + * @author Gabriel Barros + */ +public record ConversionResult(List convertedFiles, Path reportFile, Path zipFile) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java new file mode 100644 index 000000000..5e7107946 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +/** + * @author Gabriel Barros + */ +public enum JobStatus { + ACCEPTED, PROCESSING, COMPLETED, FAILED, EVICTED; +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java new file mode 100644 index 000000000..4028cd94f --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java @@ -0,0 +1,7 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +/** + * @author Gabriel Barros + */ +public record JobStatusResponse(JobStatus status) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java new file mode 100644 index 000000000..933301871 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java @@ -0,0 +1,7 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +/** + * @author Gabriel Barros + */ +public record JobSubmissionResponse(String id) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java new file mode 100644 index 000000000..80552d0b5 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java @@ -0,0 +1,17 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record ArtifactReport(@JsonProperty("logicalName") String logicalName, + @JsonProperty("originalMimeType") String originalMimeType, @JsonProperty("finalMimeType") String finalMimeType, + @JsonProperty("isBypassed") boolean isBypassed, @JsonProperty("complianceStatus") ComplianceStatus complianceStatus, + @JsonProperty("formatHistory") List formatHistory, + @JsonProperty("auditTrail") List auditTrail, @JsonProperty("errorMessage") String errorMessage) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java new file mode 100644 index 000000000..eab47c711 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java @@ -0,0 +1,17 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record AuditTrailStep(@JsonProperty("stepId") String stepId, @JsonProperty("pluginId") String pluginId, + @JsonProperty("agentName") String agentName, @JsonProperty("agentVersion") String agentVersion, + @JsonProperty("agentType") String agentType, @JsonProperty("command") String command, + @JsonProperty("parameters") Map parameters, @JsonProperty("durationMs") long durationMs, + @JsonProperty("successful") boolean successful, @JsonProperty("errorMessage") String errorMessage) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java new file mode 100644 index 000000000..8d484c56a --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +/** + * @author Gabriel Barros + */ +public enum ComplianceStatus { + PASSED, PARTIAL, FAILED +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java new file mode 100644 index 000000000..32aaec45f --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java @@ -0,0 +1,27 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.databasepreservation.modules.siard.services.conversion.model.JobStatus; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record ConversionReport(@JsonProperty("jobId") String jobId, @JsonProperty("status") JobStatus status, + @JsonProperty("originalFilename") String originalFilename, + @JsonProperty("totalArtifactsProduced") Integer totalArtifactsProduced, + @JsonProperty("artifacts") List artifacts, @JsonProperty("errorMessage") String errorMessage, + @JsonProperty("dbptkContext") DbptkContext dbptkContext) { + public ConversionReport withContext(DbptkContext context) { + return new ConversionReport(jobId, status, originalFilename, totalArtifactsProduced, artifacts, errorMessage, + context); + } + + public ConversionReport withOriginalFilename(String newFilename) { + return new ConversionReport(jobId, status, newFilename, totalArtifactsProduced, artifacts, errorMessage, + dbptkContext); + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java new file mode 100644 index 000000000..900387b56 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java @@ -0,0 +1,12 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Gabriel Barros + */ +public record DbptkContext(@JsonProperty("tableIndex") int tableIndex, @JsonProperty("rowIndex") long rowIndex, + @JsonProperty("columnIndex") int columnIndex, @JsonProperty("siardPaths") List siardPaths) { +} diff --git a/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java b/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java index 400746220..0efb41e01 100644 --- a/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java +++ b/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java @@ -34,6 +34,19 @@ protected Type getBinaryType(String typeName, int columnSize, int decimalDigits, if (typeName.contains("timestamp") || typeName.contains("rowversion")) { return super.getBinaryType(typeName, 8, decimalDigits, numPrecRadix); } + return super.getBinaryType(typeName, columnSize, decimalDigits, numPrecRadix); } + + @Override + protected Type getVarbinaryType(String typeName, int columnSize, int decimalDigits, int numPrecRadix) { + if (typeName.equalsIgnoreCase("varbinary") && (columnSize == 2147483647 || columnSize == -1)) { + Type type = new SimpleTypeBinary(columnSize); + type.setSql99TypeName("BINARY LARGE OBJECT"); + type.setSql2008TypeName("BINARY LARGE OBJECT"); + return type; + } + + return super.getVarbinaryType(typeName, columnSize, decimalDigits, numPrecRadix); + } }