Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,15 @@ default MergeGenerator mergeGenerator() {
};
}

/**
* Bulk-load statement emission ({@code CSVREAD}, {@code COPY},
* {@code LOAD DATA ...}).
*/
default org.eclipse.daanse.sql.dialect.api.generator.BulkLoadGenerator bulkLoadGenerator() {
return new org.eclipse.daanse.sql.dialect.api.generator.BulkLoadGenerator() {
};
}

/**
* Type-cast emission ({@code CAST(x AS T)}, {@code TRY_CAST},
* {@code SAFE_CAST}).
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright (c) 2026 Contributors to the Eclipse Foundation.
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* SmartCity Jena - initial
* Stefan Bischof (bipolis.org) - initial
*/
package org.eclipse.daanse.sql.dialect.api.generator;

import java.nio.file.Path;
import java.util.List;
import java.util.Optional;

import org.eclipse.daanse.sql.model.schema.TableReference;

/**
* Generates the database-native bulk-load statement for a local delimited file
* (H2 {@code CSVREAD}, DuckDB {@code read_csv}, PostgreSQL {@code COPY}, MySQL
* {@code LOAD DATA LOCAL INFILE}). The default is empty; callers fall back to
* batched INSERTs. The file path must be visible to whatever executes the
* statement.
*/
public interface BulkLoadGenerator {

/**
* The bulk-load statement for a delimited text file whose data begins after
* {@code skipLines} lines. Values are read by position; the caller has already
* created the table with {@code columns} in this order.
*
* @param skipLines lines before the first data line, headers included
* @param nullLiteral text that stands for an absent value
* @return the executable SQL statement, or empty when unsupported
*/
default Optional<String> loadFromDelimitedFile(TableReference target, List<String> columns, Path csvFile,
char delimiter, int skipLines, String nullLiteral) {
return Optional.empty();
}

/** Whether this dialect generates native bulk-load statements. */
default boolean supportsBulkLoad() {
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -218,4 +218,34 @@ public String paginate(java.util.OptionalLong limit, java.util.OptionalLong offs
return local;
}


/**
* Bulk load via {@code read_csv}. Uses {@code header=false} with {@code skip}
* because {@code header=true} would take the type line as the first data row
* and turn every column into VARCHAR.
*/
@Override
public org.eclipse.daanse.sql.dialect.api.generator.BulkLoadGenerator bulkLoadGenerator() {
return new org.eclipse.daanse.sql.dialect.api.generator.BulkLoadGenerator() {

@Override
public boolean supportsBulkLoad() {
return true;
}

@Override
public java.util.Optional<String> loadFromDelimitedFile(
org.eclipse.daanse.sql.model.schema.TableReference target, java.util.List<String> columns,
java.nio.file.Path csvFile, char delimiter, int skipLines, String nullLiteral) {
String quotedColumns = columns.stream().map(DuckDbDialect.this::quoteIdentifier)
.collect(java.util.stream.Collectors.joining(", "));
String file = csvFile.toAbsolutePath().toString().replace("'", "''");
return java.util.Optional.of("INSERT INTO " + qualified(target) + " (" + quotedColumns
+ ") SELECT * FROM read_csv('" + file + "', header=false, skip=" + skipLines + ", delim='"
+ (delimiter == '\'' ? "''" : String.valueOf(delimiter)) + "', nullstr='"
+ nullLiteral.replace("'", "''") + "')");
}
};
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -397,4 +397,5 @@ public boolean supportsPercentileCont() {
public boolean supportsNthValue() {
return true;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -265,5 +265,4 @@ public java.util.Optional<String> upsert(UpsertSpec spec, java.util.List<String>
cachedMergeGenerator = local;
return local;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,11 @@ private void loadTable(Connection connection, Path path) throws SQLException {
List<ColumnDefinition> headersTypeList = getHeadersTypeList(types);
if (it.hasNext()) {
createTable(connection, headersTypeList, tableDefinition);
insertTable(connection, it, headersTypeList, tableRef);
// skipLines = 2: the column-name line and the SQL-type line.
if (!DialectAwareLoader.loadNatively(connection, dialect, tableRef, headersTypeList, path,
config.fieldSeparator(), 2, config.nullValue())) {
insertTable(connection, it, headersTypeList, tableRef);
}
}

} catch (IOException e) {
Expand Down Expand Up @@ -241,7 +245,10 @@ public void createTable(Connection connection, List<ColumnDefinition> headersTyp
LOGGER.debug("Created table in given database. {}", sql);

stmt.execute(sql);
connection.commit();
if (!connection.getAutoCommit()) {
// commit() on an auto-commit connection is a JDBC error; DuckDB rejects it.
connection.commit();
}
} catch (SQLException e) {
throw new CsvDataImporterException("Exception wile create table", e);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright (c) 2026 Contributors to the Eclipse Foundation.
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* SmartCity Jena - initial
* Stefan Bischof (bipolis.org) - initial
*/
package org.eclipse.daanse.sql.jdbc.importer.csv.impl;

import java.nio.file.Path;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Optional;

import org.eclipse.daanse.sql.dialect.api.Dialect;
import org.eclipse.daanse.sql.model.schema.ColumnDefinition;
import org.eclipse.daanse.sql.model.schema.TableReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Loads a delimited file with the dialect's native bulk-load statement, if it
* has one; otherwise the caller falls back to batched INSERTs.
*/
final class DialectAwareLoader {

private static final Logger LOGGER = LoggerFactory.getLogger(DialectAwareLoader.class);

private DialectAwareLoader() {
// static access only
}

/**
* @param skipLines lines before the first data line
* @return whether the file was loaded; {@code false} means the caller has to
* do it row by row
*/
static boolean loadNatively(Connection connection, Dialect dialect, TableReference target,
List<ColumnDefinition> columns, Path file, char delimiter, int skipLines, String nullLiteral)
throws SQLException {
if (!dialect.bulkLoadGenerator().supportsBulkLoad()) {
return false;
}
Optional<String> statement = dialect.bulkLoadGenerator().loadFromDelimitedFile(target,
columns.stream().map(column -> column.column().name()).toList(), file, delimiter, skipLines, nullLiteral);
if (statement.isEmpty()) {
LOGGER.debug("{} has no bulk load for a file with {} leading lines; loading {} row by row",
dialect.name(), skipLines, file.getFileName());
return false;
}
long started = System.currentTimeMillis();
try (Statement direct = connection.createStatement()) {
direct.execute(statement.get());
} catch (SQLException e) {
// Typically the server cannot see the file or local reads are off.
LOGGER.warn("{} refused to read {} itself ({}); loading row by row", dialect.name(), file.getFileName(),
e.getMessage());
// Truncate whatever the failed attempt left, or the fallback load adds up.
try (Statement direct = connection.createStatement()) {
direct.executeUpdate(dialect.ddlGenerator().truncate(target));
}
return false;
}
LOGGER.info("{} read {} itself in {} ms", dialect.name(), file.getFileName(),
System.currentTimeMillis() - started);
return true;
}
}
Loading