Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ image::hop-tools/setup/06-configure-hop-environment-dialog-projects-tab.png[Conf
** Windows: `%USERPROFILE%\Documents\Hop\default`
* Register the *samples* project from this Hop installation (`<install>/config/projects/samples`) so the examples stay available after the config folder moves.

Both options are selected by default. Apply writes them into the new `hop-config.json` (and creates `project-config.json` plus the install default-project template when the default home is empty).
Both options are selected by default. Apply writes them into the new `hop-config.json`. An empty default project home is created from the install default-project template. That project includes a `local` pipeline run configuration and a `local` workflow run configuration. If the template does not contain them, setup adds them, together with the local execution info location and data profile they use.

=== Preview and apply

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ public void seed(HopEnvironmentSpec spec, HopEnvironmentApplyResult result, Path
result.addMessage("Would copy " + source + " to " + spec.getConfigFolder());
return;
}
HopVfsFiles.copyTree(source, spec.getConfigFolder());
// One level only: nested projects/default and projects/samples stay in the install tree.
HopVfsFiles.copyChildren(source, spec.getConfigFolder());
result.addMessage("Copied install config from " + source + " to " + spec.getConfigFolder());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSelector;
import org.apache.commons.vfs2.Selectors;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.setup.HopSetupException;
Expand Down Expand Up @@ -72,14 +73,29 @@ public static void createFolder(String path) throws HopSetupException {
}
}

/**
* Copies the direct children of {@code sourcePath} into {@code destinationPath}. Child folders
* are created without their contents.
*/
public static void copyChildren(String sourcePath, String destinationPath)
throws HopSetupException {
copySelected(sourcePath, destinationPath, Selectors.SELECT_CHILDREN);
}

/** Copies every file and subfolder under {@code sourcePath} into {@code destinationPath}. */
public static void copyTree(String sourcePath, String destinationPath) throws HopSetupException {
copySelected(sourcePath, destinationPath, Selectors.EXCLUDE_SELF);
}

private static void copySelected(String sourcePath, String destinationPath, FileSelector selector)
throws HopSetupException {
try {
FileObject source = HopVfs.getFileObject(sourcePath);
FileObject destination = HopVfs.getFileObject(destinationPath);
if (!destination.exists()) {
destination.createFolder();
}
destination.copyFrom(source, Selectors.SELECT_CHILDREN);
destination.copyFrom(source, selector);
} catch (Exception e) {
throw new HopSetupException(
"Unable to copy '" + sourcePath + "' to '" + destinationPath + "'", e);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hop.setup.persist;

import org.apache.commons.lang3.StringUtils;
import org.apache.hop.setup.HopEnvironmentApplyResult;
import org.apache.hop.setup.HopSetupException;

/**
* Adds the standard {@code local} pipeline and workflow run configurations to a new project.
*
* <p>Files already present are left unchanged. A file that exists in the install default-project
* template is copied from there. Otherwise the stock default-project content is written, including
* the local execution info location and data profile those run configurations use.
*/
public class LocalRunConfigurationSeeder {

static final String PIPELINE_RELATIVE = "metadata/pipeline-run-configuration/local.json";
static final String WORKFLOW_RELATIVE = "metadata/workflow-run-configuration/local.json";
static final String EXECUTION_INFO_RELATIVE = "metadata/execution-info-location/local-audit.json";
static final String DATA_PROFILE_RELATIVE = "metadata/execution-data-profile/first-last.json";

static final String PIPELINE_JSON =
"""
{
"engineRunConfiguration": {
"Local": {
"feedback_size": "50000",
"sample_size": "100",
"sample_type_in_gui": "Last",
"rowset_size": "10000",
"safe_mode": false,
"show_feedback": false,
"topo_sort": false,
"gather_metrics": false
}
},
"name": "local",
"configurationVariables": [],
"description": "",
"executionInfoLocationName": "local-audit",
"dataProfile": "first-last",
"defaultSelection": true
}
""";

static final String WORKFLOW_JSON =
"""
{
"engineRunConfiguration": {
"Local": {
"safe_mode": false
}
},
"name": "local",
"description": "",
"executionInfoLocationName": "local-audit",
"defaultSelection": true
}
""";

static final String EXECUTION_INFO_JSON =
"""
{
"executionInfoLocation": {
"local-folder": {
"pluginName": "File location",
"pluginId": "local-folder",
"rootFolder": "${HOP_AUDIT_FOLDER}/executions/"
}
},
"dataLoggingDelay": "2000",
"name": "local-audit",
"description": "",
"dataLoggingInterval": "5000"
}
""";

static final String DATA_PROFILE_JSON =
"""
{
"name": "first-last",
"description": "capture the first and last 100 rows of every Hop transform",
"sampler": [
{
"FirstRowsExecutionDataSampler": {
"sampleSize": "100"
}
},
{
"LastRowsExecutionDataSampler": {
"sampleSize": "100"
}
}
]
}
""";

public boolean standardRunConfigurationsMissing(String templateHome) throws HopSetupException {
return !templateContains(templateHome, PIPELINE_RELATIVE)
|| !templateContains(templateHome, WORKFLOW_RELATIVE);
}

public void ensure(String projectHome, String templateHome, HopEnvironmentApplyResult result)
throws HopSetupException {
boolean synthesized =
ensureRunConfiguration(
projectHome,
templateHome,
PIPELINE_RELATIVE,
PIPELINE_JSON,
"local pipeline run configuration",
result);
synthesized =
ensureRunConfiguration(
projectHome,
templateHome,
WORKFLOW_RELATIVE,
WORKFLOW_JSON,
"local workflow run configuration",
result)
|| synthesized;
if (synthesized) {
ensureFile(
projectHome,
templateHome,
EXECUTION_INFO_RELATIVE,
EXECUTION_INFO_JSON,
"local execution info location",
result);
ensureFile(
projectHome,
templateHome,
DATA_PROFILE_RELATIVE,
DATA_PROFILE_JSON,
"execution data profile",
result);
} else {
copyFromTemplateIfMissing(
projectHome,
templateHome,
EXECUTION_INFO_RELATIVE,
"local execution info location",
result);
copyFromTemplateIfMissing(
projectHome, templateHome, DATA_PROFILE_RELATIVE, "execution data profile", result);
}
}

private boolean ensureRunConfiguration(
String projectHome,
String templateHome,
String relative,
String fallback,
String label,
HopEnvironmentApplyResult result)
throws HopSetupException {
String destination = projectHome + "/" + relative;
if (HopVfsFiles.exists(destination)) {
return false;
}
if (templateContains(templateHome, relative)) {
write(destination, HopVfsFiles.readUtf8(templateHome + "/" + relative), label, result);
return false;
}
write(destination, fallback, label, result);
return true;
}

private void ensureFile(
String projectHome,
String templateHome,
String relative,
String fallback,
String label,
HopEnvironmentApplyResult result)
throws HopSetupException {
String destination = projectHome + "/" + relative;
if (HopVfsFiles.exists(destination)) {
return;
}
String content =
templateContains(templateHome, relative)
? HopVfsFiles.readUtf8(templateHome + "/" + relative)
: fallback;
write(destination, content, label, result);
}

private void copyFromTemplateIfMissing(
String projectHome,
String templateHome,
String relative,
String label,
HopEnvironmentApplyResult result)
throws HopSetupException {
String destination = projectHome + "/" + relative;
if (HopVfsFiles.exists(destination) || !templateContains(templateHome, relative)) {
return;
}
write(destination, HopVfsFiles.readUtf8(templateHome + "/" + relative), label, result);
}

private static boolean templateContains(String templateHome, String relative)
throws HopSetupException {
return StringUtils.isNotBlank(templateHome)
&& HopVfsFiles.exists(templateHome + "/" + relative);
}

private static void write(
String destination, String content, String label, HopEnvironmentApplyResult result)
throws HopSetupException {
HopVfsFiles.writeUtf8(destination, content);
result.addMessage("Created " + label + " at " + destination);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,10 @@

/**
* Creates a user-owned default project and registers it (plus the install samples project) in the
* target {@code hop-config.json}. Does not use the live {@code HopConfig} singleton, which still
* points at the current process config folder.
* target {@code hop-config.json}. A new default project gets a {@code local} pipeline run
* configuration and a {@code local} workflow run configuration when it does not already have them.
* Does not use the live {@code HopConfig} singleton, which still points at the current process
* config folder.
*/
public class ProjectRegistrationWriter {

Expand All @@ -55,10 +57,17 @@ public class ProjectRegistrationWriter {
""";

private final ObjectMapper mapper;
private final LocalRunConfigurationSeeder runConfigurations;

public ProjectRegistrationWriter() {
this(new LocalRunConfigurationSeeder());
}

ProjectRegistrationWriter(LocalRunConfigurationSeeder runConfigurations) {
mapper = HopJson.newMapper();
mapper.enable(SerializationFeature.INDENT_OUTPUT);
this.runConfigurations =
runConfigurations == null ? new LocalRunConfigurationSeeder() : runConfigurations;
}

public void apply(
Expand Down Expand Up @@ -118,6 +127,10 @@ private void seedDefaultProject(
hasTemplate
? "Would copy default project template to " + defaultHome
: "Would create default project at " + defaultHome);
if (runConfigurations.standardRunConfigurationsMissing(hasTemplate ? template : null)) {
result.addMessage(
"Would add local pipeline and workflow run configurations to " + defaultHome);
}
return;
}
HopVfsFiles.createFolder(defaultHome);
Expand All @@ -128,6 +141,7 @@ private void seedDefaultProject(
HopVfsFiles.writeUtf8(projectConfig, MINIMAL_PROJECT_CONFIG);
result.addMessage("Created default project at " + defaultHome);
}
runConfigurations.ensure(defaultHome, hasTemplate ? template : null, result);
}

private String resolveSamplesHome(Path hopHome, HopEnvironmentApplyResult result)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ SetupDialog.Web.ReadOnly=Hop Web cannot write the server operating system enviro
SetupDialog.Tab.Variables=Variables
SetupDialog.Tab.Location=Location
SetupDialog.Tab.Projects=Projects
SetupDialog.Projects.Intro=After you point Hop at a new config folder it starts without projects. Create a default project in your documents folder and register the samples project from this Hop installation.
SetupDialog.Projects.Intro=After you point Hop at a new config folder it starts without projects. Create a default project in your documents folder, including a local pipeline and workflow run configuration, and register the samples project from this Hop installation.
SetupDialog.Projects.CreateDefault=Create a default project
SetupDialog.Projects.DefaultHome.Label=Default project home
SetupDialog.Projects.Recommended=Use recommended project folder
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,17 @@ void copiesInstallConfigWhenTargetEmpty() throws Exception {
Files.createDirectories(install.resolve("config"));
Files.writeString(install.resolve("config").resolve(HopSetupVariables.HOP_CONFIG_JSON), "{}");
Files.writeString(install.resolve("config").resolve("extra.txt"), "keep");
Path defaultProject =
install
.resolve("config/projects/default/metadata/pipeline-run-configuration")
.resolve("local.json");
Files.createDirectories(defaultProject.getParent());
Files.writeString(defaultProject, "{ \"name\": \"local\" }");
Path samplesProject = install.resolve("config/projects/samples");
Files.createDirectories(samplesProject);
Files.writeString(
samplesProject.resolve(HopSetupVariables.PROJECT_CONFIG_FILENAME),
"{ \"description\": \"samples\" }");
Path target = temp.resolve("user-config");
HopEnvironmentSpec spec = new HopEnvironmentSpec();
spec.setConfigFolder(target.toString());
Expand All @@ -60,6 +71,9 @@ void copiesInstallConfigWhenTargetEmpty() throws Exception {
new ConfigFolderSeeder().seed(spec, new HopEnvironmentApplyResult(), install);
assertTrue(Files.exists(target.resolve(HopSetupVariables.HOP_CONFIG_JSON)));
assertTrue(Files.exists(target.resolve("extra.txt")));
assertTrue(Files.isDirectory(target.resolve("projects")));
assertFalse(Files.exists(target.resolve("projects/default")));
assertFalse(Files.exists(target.resolve("projects/samples")));
}

@Test
Expand Down
Loading
Loading