Issue #8671 : Read and write Windows shares through an SMB VFS connection

assemblies/plugins/pom.xml

@@ -756,6 +756,12 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hop</groupId>
+            <artifactId>hop-tech-smb</artifactId>
+            <version>${project.version}</version>
+            <type>zip</type>
+        </dependency>
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-tech-vault</artifactId>

assemblies/static/src/main/resources/LICENSE

@@ -517,7 +517,7 @@ This product bundles batik-xml 1.19, which is available under a
 This product bundles bcjmail-jdk18on 1.84, which is available under a
 "Bouncy Castle Licence" license. For details, see licenses/LICENSE-BOUNCY.
 
-This product bundles bcprov-jdk18on 1.85.2, which is available under a
+This product bundles bcprov-jdk18on 1.86, which is available under a
 "Bouncy Castle Licence" license. For details, see licenses/LICENSE-BOUNCY.
 
 This product bundles beam-model-fn-execution 2.74.0, which is available under a

core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java

@@ -126,6 +126,7 @@ public enum HopMetadataPropertyType {
   VFS_SFTP_CONNECTION,
   VFS_FTP_CONNECTION,
   VFS_HDFS_CONNECTION,
+  VFS_SMB_CONNECTION,
   VFS_GIT_CONNECTION,
 
   // DATA STREAM

docker/integration-tests/integration-tests-smb.yaml

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+  integration_test_smb:
+    extends:
+      file: integration-tests-base.yaml
+      service: integration_test
+    depends_on:
+      samba:
+        condition: service_healthy
+      samba-encrypted:
+        condition: service_healthy
+
+  samba:
+    build:
+      context: ./resource/smb
+    hostname: samba
+    environment:
+      SMB_PASSWORD: hop-it-password
+    healthcheck:
+      test: ["CMD", "smbclient", "-L", "127.0.0.1", "-U", "hop%hop-it-password", "-m", "SMB3"]
+      interval: 5s
+      timeout: 10s
+      retries: 20
+      start_period: 5s
+
+  samba-encrypted:
+    build:
+      context: ./resource/smb
+    hostname: samba-encrypted
+    environment:
+      SMB_PASSWORD: hop-it-password
+      SMB_ENCRYPT: required
+    healthcheck:
+      test: ["CMD", "smbclient", "-L", "127.0.0.1", "-U", "hop%hop-it-password", "-m", "SMB3"]
+      interval: 5s
+      timeout: 10s
+      retries: 20
+      start_period: 5s

docker/integration-tests/resource/smb/Dockerfile

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM debian:bookworm-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends samba smbclient \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY smb.conf /etc/samba/smb.conf
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+
+EXPOSE 445
+ENTRYPOINT ["/entrypoint.sh"]

docker/integration-tests/resource/smb/entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+if [ "${SMB_ENCRYPT:-}" = "required" ]; then
+  sed -i 's/smb encrypt = desired/smb encrypt = required/' /etc/samba/smb.conf
+fi
+
+id hop >/dev/null 2>&1 || useradd -M -s /usr/sbin/nologin hop
+printf '%s\n%s\n' "$SMB_PASSWORD" "$SMB_PASSWORD" | smbpasswd -a -s hop
+mkdir -p /share
+chown hop:hop /share
+exec smbd -F --no-process-group --debug-stdout

docker/integration-tests/resource/smb/smb.conf

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[global]
+   workgroup = WORKGROUP
+   server role = standalone server
+   security = user
+   server min protocol = SMB2
+   server max protocol = SMB3
+   ntlm auth = ntlmv2-only
+   map to guest = Never
+   usershare allow guests = no
+   server signing = mandatory
+   smb encrypt = desired
+   disable netbios = yes
+   smb ports = 445
+   log level = 1
+   bind interfaces only = no
+
+[data]
+   path = /share
+   read only = no
+   browsable = yes
+   valid users = hop
+   force user = hop
+   create mask = 0644
+   directory mask = 0755

docs/hop-dev-manual/modules/ROOT/pages/plugin-types/data-connectivity.adoc

@@ -95,6 +95,8 @@ The two stream classes are thin wrappers extending `CompressionInputStream` / `C
 Hop addresses every file through Apache VFS, so adding support for a new kind of storage means adding a VFS provider.
 This is how `s3://`, `azure://`, `gs://`, `ftp://` and `sftp://` work.
 HDFS is a named VFS connection (marketplace plugin `hop-tech-hadoop`); it does not register a global `hdfs://` scheme, because Native Spark already uses `hdfs://` as a Hadoop FileSystem URI.
+SMB (`plugins/tech/smb`) is also a named connection only.
+It uses smbj for SMB 2 and SMB 3, does not register `smb://`, and does not speak SMB1.
 
 Annotation:: `@VfsPlugin`
 Implement:: `IVfs`

docs/hop-user-manual/modules/ROOT/nav.adoc

@@ -452,6 +452,7 @@ under the License.
 ** xref:vfs/git-vfs.adoc[Git]
 ** xref:vfs/hdfs-vfs.adoc[HDFS]
 ** xref:vfs/minio-vfs.adoc[MinIO]
+** xref:vfs/smb-vfs.adoc[SMB]
 * xref:database/databases.adoc[Relational Database Connections]
 //::=START AUTO GENERATED LINKS DATABASES
 ** xref:database/databases/derby.adoc[Apache Derby]
@@ -543,6 +544,7 @@ under the License.
 ** xref:metadata-types/rest-connection.adoc[REST Connection]
 ** xref:metadata-types/salesforce-connection.adoc[Salesforce Connection]
 ** xref:metadata-types/sftp-connection.adoc[SFTP Connection]
+** xref:metadata-types/smb-connection.adoc[SMB Connection]
 ** xref:metadata-types/splunk-connection.adoc[Splunk Connection]
 ** xref:metadata-types/jms-connection.adoc[JMS connection]
 ** xref:metadata-types/spark-catalog.adoc[Spark Catalog]

docs/hop-user-manual/modules/ROOT/pages/metadata-types/smb-connection.adoc

@@ -0,0 +1,77 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:imagesdir: ../../assets/images/
+:page-pagination:
+:description: Named SMB2/SMB3 connection
+
+= SMB Connection
+
+== Description
+
+This metadata type registers a Windows or Samba file share as a named VFS connection.
+The connection name is the URL scheme, for example `finance:///reports/daily.csv`.
+
+The plugin talks SMB 2 and SMB 3 directly.
+Nothing is mounted on the operating system, and SMB1 is not used.
+
+See xref:vfs/smb-vfs.adoc[SMB] for how paths are written.
+
+image::metadata-types/smb-connection.png[SMB connection editor on the Server tab, with name finance, width="90%"]
+
+The name at the top of the editor is the URL scheme.
+The tabs are Server, Authentication, and Security.
+*Test connection* opens the share with the current settings.
+
+== Server
+
+[options="header"]
+|===
+|Option|Description
+|Description|Optional note about this connection. It is not part of the URL.
+|Hostname|Server name. A variable such as `${FINANCE_FILE_SERVER}` is resolved at runtime. For Azure Files this is `account.file.core.windows.net`.
+|Port|TCP port. Default `445`.
+|Share|Share name, without a leading slash.
+|Folder in the share|Optional folder. Every path on this connection stays under that folder.
+|===
+
+== Authentication
+
+[options="header"]
+|===
+|Option|Description
+|NTLM|Username, password, and an optional NetBIOS domain. This is NTLMv2. A username of `DOMAIN\user` is split when Domain is empty. `user@dns.domain` is not parsed: put the domain in Domain and the sAMAccountName in Username.
+|GUEST|Ignores username and password. Current Windows has guest SMB2 access off by default.
+|===
+
+Kerberos is not available in this version.
+The connection type is the place it will be added.
+
+The password is stored with Hop's password encoder.
+A variable such as `${SMB_PASSWORD}` is resolved at runtime.
+
+== Security
+
+[options="header"]
+|===
+|Option|Description
+|Minimum SMB version|Lowest dialect to offer. The client never goes above SMB 3.1.1 and never offers SMB1.
+|Require SMB signing|Refuse a server that does not sign. When this is off, the client still signs if the server requires it.
+|Encrypt session|Require SMB3 encryption.
+|Follow DFS referrals|Follow Distributed File System referrals. Leave off unless the share is a DFS root.
+|Call timeout|Read, write, and transact timeout in seconds. Default 60.
+|Socket timeout|How long the background reader waits for the next packet, in seconds. 0 waits until the server sends one or drops the connection. Leave this at 0. A positive value closes a quiet session.
+|===

docs/hop-user-manual/modules/ROOT/pages/vfs.adoc

@@ -52,6 +52,7 @@ Click the File system name to access more detailed file system documentation.
 |xref:vfs/google-drive-vfs.adoc[Google Drive]|Provides access to Google Drive folders|`googledrive://`
 |xref:vfs/git-vfs.adoc[Git]|Named connection to a git repository checked out at a branch, tag or commit (marketplace plugin `hop-tech-git-vfs`)|`+<connectionName>:///path+`
 |xref:vfs/hdfs-vfs.adoc[HDFS]|Named connection to HDFS via HttpFS, Knox or WebHDFS (marketplace plugin `hop-tech-hadoop`)|`+<connectionName>:///path+`
+|xref:vfs/smb-vfs.adoc[SMB]|Named connection to an SMB2/SMB3 file share|`+<connectionName>:///path+`
 |xref:metadata-types/minio-connection.adoc[Minio connection]|Provides access to S3-compatible endpoints via a named Minio connection (metadata)|`+<connectionName>://+`
 |xref:metadata-types/webdav-connection.adoc[WebDAV Connection]|Provides access to WebDAV servers, either via the static schemes or a named connection (metadata)|`webdav4://`, `webdav4s://`, or `+<connectionName>://+`
 |===

docs/hop-user-manual/modules/ROOT/pages/vfs/smb-vfs.adoc

@@ -0,0 +1,58 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+
+:documentationPath: /vfs/
+:language: en_US
+:description: SMB2 and SMB3 file shares through a named VFS connection
+
+= SMB
+
+== Scheme
+
+SMB is a named VFS connection.
+The scheme is the connection name:
+
+`finance:///reports/2026/daily.csv`
+
+Host, share, and credentials live on the xref:metadata-types/smb-connection.adoc[SMB Connection].
+They are not part of the URL.
+A UNC path such as `\\server\share` is not a VFS URI, and Hop does not register `smb://`.
+
+Text File Input, Text File Output, Copy Files, and the VFS File Explorer use the connection directly.
+There is no separate Get or Put action.
+
+== What it speaks
+
+The client is SMB 2.0.2 through SMB 3.1.1.
+It does not speak SMB1.
+That matches current Windows, where SMB1 is off.
+
+Sign-in for this version is NTLMv2 with a username and password, plus an optional domain.
+Guest is available for a lab share that still allows it.
+Current Windows has guest SMB2 access off by default.
+Kerberos is not implemented yet.
+
+Azure Files is the same NTLM connection: hostname `account.file.core.windows.net`, username is the storage account, password is the access key, and the share is the file share name.
+
+== Check against Windows
+
+The automated tests run against Samba with SMB1 disabled, NTLMv2 only, and SMB signing required.
+To confirm a real Windows share:
+
+. Create an xref:metadata-types/smb-connection.adoc[SMB Connection] aimed at a server that has SMB1 turned off.
+. Use *Test connection*.
+. Write a small file with Text File Output to `connectionName:///folder/file.csv` and read it back.

integration-tests/smb/dev-env-config.json

@@ -0,0 +1,31 @@
+{
+  "variables" : [ {
+    "name" : "SMB_HOSTNAME",
+    "value" : "samba",
+    "description" : "Compose service name of the signing Samba server"
+  }, {
+    "name" : "SMB_ENCRYPTED_HOSTNAME",
+    "value" : "samba-encrypted",
+    "description" : "Compose service name of the Samba server that requires encryption"
+  }, {
+    "name" : "SMB_PORT",
+    "value" : "445",
+    "description" : ""
+  }, {
+    "name" : "SMB_USER",
+    "value" : "hop",
+    "description" : ""
+  }, {
+    "name" : "SMB_PASSWORD",
+    "value" : "hop-it-password",
+    "description" : "Matches the Samba container. A variable is not encrypted into the connection JSON."
+  }, {
+    "name" : "SMB_BAD_PASSWORD",
+    "value" : "not-the-password",
+    "description" : ""
+  }, {
+    "name" : "SMB_SHARE",
+    "value" : "data",
+    "description" : ""
+  } ]
+}