From 319d86e59c817e2b83c562d4b0a4b510522e10f2 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Tue, 25 Jun 2013 00:11:18 +0000 Subject: [PATCH 1/2] Fixes to build_release.sh script * storm-console-processor was letting clojure-1.2 meet its dependency list, making for two clojure jars in the tarball. Added a dependency to its project.clj * storm-netty had a dependency itself on storm-core, so the storm-core jar was in both _release and _release/lib. removing the dupe jar from lib/ * build_releases should fail if the build_modules script fails. Made build_modules run set -e, and made build_releases fail if it does * maven is putting the storm-* jar files in target/release+provided, not target/provided. Fixed path. * made pretty banners, made paths be properly quoted * bin/ directory files copy with permissions intact. Also now remove build_release.sh and build_modules.sh, since they won't work * rather than remove _release and *.zip, only removes the specific subdirectory and .zip file. If you run with STORM_KEEP_RELEASE=true, it will not delete the release dir at the end of script (only at beginning) * bin/storm tells you how to roll your own release if you try to launch from a git checkout --- bin/build_modules.sh | 39 +++++++------ bin/build_release.sh | 95 +++++++++++++++++++------------ bin/storm | 1 + storm-console-logging/project.clj | 2 + 4 files changed, 86 insertions(+), 51 deletions(-) mode change 100644 => 100755 bin/build_modules.sh mode change 100644 => 100755 bin/build_release.sh diff --git a/bin/build_modules.sh b/bin/build_modules.sh old mode 100644 new mode 100755 index 44d62ba4e2..805aa5bd8c --- a/bin/build_modules.sh +++ b/bin/build_modules.sh @@ -1,25 +1,32 @@ #!/bin/bash -LEIN=`which lein2 || which lein` +set -e # die if anything bad happens + +function banner { + echo ; echo "------------- $1 ---------------" ; echo +} + +LEIN=`which lein2 || which lein` export LEIN_ROOT=1 for module in $(cat MODULES) do - echo "Building $module" - cd $module - if [ $module != "storm-console-logging" ] - then - rm ../conf/logback.xml - fi - + banner "Building $module" + cd $module + if [ $module != "storm-console-logging" ] + then + rm -f ../conf/logback.xml + else + git checkout ../conf/logback.xml ; true + fi - $LEIN with-profile release clean - $LEIN with-profile release deps - $LEIN with-profile release jar - $LEIN with-profile release install - $LEIN with-profile release pom + $LEIN with-profile release clean + $LEIN with-profile release deps + $LEIN with-profile release jar + $LEIN with-profile release install + $LEIN with-profile release pom - git checkout ../conf/logback.xml - cd .. -done \ No newline at end of file + git checkout ../conf/logback.xml ; true + cd .. +done diff --git a/bin/build_release.sh b/bin/build_release.sh old mode 100644 new mode 100755 index 723b2f4a8c..86a078c08e --- a/bin/build_release.sh +++ b/bin/build_release.sh @@ -1,62 +1,87 @@ #!/bin/bash function quit { - exit 1 + exit 1 +} +trap quit 1 2 3 15 # Ctrl+C exits. + +function banner { + echo ; echo "=========================" ; echo "==" ; echo "== $1" ; echo "==" ; echo } -trap quit 1 2 3 15 #Ctrl+C exits. RELEASE=`cat VERSION` -LEIN=`which lein2 || which lein` +LEIN=`which lein2 || which lein` export LEIN_ROOT=1 -echo Making release $RELEASE +# ========================================================================== +banner "Making release $RELEASE" -DIR=`pwd`/_release/storm-$RELEASE +CODE_ROOT=`pwd` +DIR="$CODE_ROOT/_release/storm-$RELEASE" +ZIPFILE="$CODE_ROOT/storm-$RELEASE.zip" -rm -rf _release -rm -f *.zip +rm -rf "$DIR" +rm -f "$ZIPFILE" $LEIN pom || exit 1 -mkdir -p $DIR/lib +mkdir -p "$DIR/lib" + +# ========================================================================== +banner "Building submodules" +/bin/bash "$CODE_ROOT/bin/build_modules.sh" || exit 1 -sh bin/build_modules.sh +# ========================================================================== +banner "Gathering dependencies" for module in $(cat MODULES) do - cd $module - mvn dependency:copy-dependencies || exit 1 - cp -f target/dependency/*.jar $DIR/lib/ - cp -f target/*.jar $DIR/ - cd .. + cd "$CODE_ROOT/$module" + mvn dependency:copy-dependencies || exit 1 + cp -f target/dependency/*.jar "$DIR/lib/" + cp -f target/*.jar "$DIR/" + cd "$CODE_ROOT" done -cd _release/storm-$RELEASE -for i in *.jar +# The netty libs have storm itself as a dependency; remove any jar in $DIR/lib/ that is in $DIR/ +cd "$DIR" +for base_jar in *.jar do - rm -f lib/$i -done -cd ../.. + rm -f lib/$base_jar +done +cd "$CODE_ROOT" + +# ========================================================================== +banner "Copying support files" + +cp -v CHANGELOG.md "$DIR/" -cp CHANGELOG.md $DIR/ +echo $RELEASE > "$DIR/RELEASE" -echo $RELEASE > $DIR/RELEASE +mkdir -p "$DIR/logback" +mkdir -p "$DIR/logs" +cp -vR logback/cluster.xml "$DIR/logback/cluster.xml" -mkdir -p $DIR/logback -mkdir -p $DIR/logs -cp -R logback/cluster.xml $DIR/logback/cluster.xml +mkdir "$DIR/conf" +cp -v conf/storm.yaml.example "$DIR/conf/storm.yaml" -mkdir $DIR/conf -cp conf/storm.yaml.example $DIR/conf/storm.yaml +cp -vR storm-core/src/ui/public "$DIR/" -cp -R storm-core/src/ui/public $DIR/ +cp -vRp bin "$DIR/" +rm "$DIR"/bin/build_{release,modules}.sh -cp -R bin $DIR/ +cp -v README.markdown "$DIR/" +cp -v LICENSE.html "$DIR/" -cp README.markdown $DIR/ -cp LICENSE.html $DIR/ +# ========================================================================== +banner "Building Zip File in '$ZIPFILE'" -cd _release -zip -r storm-$RELEASE.zip * -cd .. -mv _release/storm-*.zip . -rm -rf _release +cd "$CODE_ROOT/_release" +zip -r "$ZIPFILE" * +cd "$CODE_ROOT" +echo +if [ "$STORM_KEEP_RELEASE" == "true" ] ; then + echo "keeping _release dir '$DIR'" +else + echo "removing _release dir" + rm -rf "$DIR" +fi diff --git a/bin/storm b/bin/storm index 6960bc9201..d73d9d0eb0 100755 --- a/bin/storm +++ b/bin/storm @@ -39,6 +39,7 @@ if not os.path.exists(STORM_DIR + "/RELEASE"): print "******************************************" print "The storm client can only be run from within a release. You appear to be trying to run the client from a checkout of Storm's source code." print "\nYou can download a Storm release at https://github.com/nathanmarz/storm/downloads" + print "\nor build it yourself by running ./bin/build_release.sh from a clone of the source repo." print "******************************************" sys.exit(1) diff --git a/storm-console-logging/project.clj b/storm-console-logging/project.clj index c199244dc3..79d9d4f221 100644 --- a/storm-console-logging/project.clj +++ b/storm-console-logging/project.clj @@ -2,6 +2,8 @@ (def VERSION (-> ROOT-DIR (str "/../VERSION") slurp (.trim))) (defproject storm/storm-console-logging VERSION + :dependencies [[org.clojure/clojure "1.4.0"] + ] :resource-paths ["logback"] :profiles {:release {} From 93f49a3d824d605ceb6581d5d333ca06e1a47815 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Sat, 13 Jul 2013 19:08:44 +0000 Subject: [PATCH 2/2] Don't let leiningen override the target path with target/release Recent leiningen (technomancy/leiningen@a1d7b6f3eabe4eb) appends the profile name to the target directory -- if you installed lein before may, you got a pom with build dir = target/; recent leins set it to target/{profile_name}. This expressly fixes it to be just plain 'target'. --- storm-console-logging/project.clj | 4 ++-- storm-core/project.clj | 2 +- storm-netty/project.clj | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/storm-console-logging/project.clj b/storm-console-logging/project.clj index 79d9d4f221..98e94bf759 100644 --- a/storm-console-logging/project.clj +++ b/storm-console-logging/project.clj @@ -6,7 +6,7 @@ ] :resource-paths ["logback"] - :profiles {:release {} + :profiles { + :release { :target-path "target" } } - :aot :all) diff --git a/storm-core/project.clj b/storm-core/project.clj index 0a21d3f6ca..a05f24d419 100644 --- a/storm-core/project.clj +++ b/storm-core/project.clj @@ -36,7 +36,7 @@ :profiles {:dev {:resource-paths ["src/dev"] :dependencies [[org.mockito/mockito-all "1.9.5"]]} - :release {} + :release { :target-path "target" } :lib {} } diff --git a/storm-netty/project.clj b/storm-netty/project.clj index 3f323ce1ee..765e2bc71c 100644 --- a/storm-netty/project.clj +++ b/storm-netty/project.clj @@ -6,6 +6,8 @@ [io.netty/netty "3.6.3.Final"]] :java-source-paths ["src/jvm"] :test-paths ["test/clj"] - :profiles {:release {}} + :profiles { + :release { :target-path "target" } + } :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] :aot :all))